diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
deleted file mode 100644
index 9197c81a..00000000
--- a/.github/workflows/build.yml
+++ /dev/null
@@ -1,18 +0,0 @@
-name: Build native artifacts
-
-on:
- workflow_dispatch:
- pull_request:
-
-jobs:
- cuda:
- uses: CAVISE/pipelines/.github/workflows/python-cmake-cuda.yml@main
- with:
- requirements-file: requirements-cuda.txt
- cuda-architectures: "86"
- pythonpath-subdir: OpenCOOD
- python-import-modules: >-
- ["opencood.pcdet_utils.iou3d_nms.iou3d_nms_cuda",
- "opencood.pcdet_utils.pointnet2.pointnet2_batch.pointnet2_batch_cuda",
- "opencood.pcdet_utils.pointnet2.pointnet2_stack.pointnet2_stack_cuda",
- "opencood.pcdet_utils.roiaware_pool3d.roiaware_pool3d_cuda"]
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 442edc51..710bac6d 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -23,14 +23,13 @@ repos:
- id: check-json
- id: check-merge-conflict
args: ["--assume-in-merge"]
- exclude: '(docs/.*|OpenCOOD/docs/.*)'
+ exclude: 'docs/.*'
- id: check-shebang-scripts-are-executable
- id: check-symlinks
- id: check-toml
- id: check-vcs-permalinks
- id: check-xml
- id: check-yaml
- exclude: opencda/coperception_models/.*
- id: destroyed-symlinks
- id: end-of-file-fixer
- id: name-tests-test
@@ -46,6 +45,8 @@ repos:
description: ''
entry: mypy
language: python
+ additional_dependencies:
+ - mypy
'types_or': [python, pyi]
args: [".", "--follow-imports=silent"]
pass_filenames: false
diff --git a/AIM/models/mtp/GNN_mtl_gnn/GNN_mtl_gnn.py b/AIM/models/mtp/GNN_mtl_gnn/GNN_mtl_gnn.py
index 49bc8fab..26d395e7 100644
--- a/AIM/models/mtp/GNN_mtl_gnn/GNN_mtl_gnn.py
+++ b/AIM/models/mtp/GNN_mtl_gnn/GNN_mtl_gnn.py
@@ -31,7 +31,7 @@ def __init__(self, hidden_channels: int) -> None:
self.linear4: nn.Linear = nn.Linear(hidden_channels, hidden_channels)
self.linear5: nn.Linear = nn.Linear(hidden_channels, 30 * 2)
- def forward(
+ def forward( # noqa: DC04 - invoked dynamically by torch.nn.Module.__call__
self,
x: torch.Tensor,
edge_index: torch.Tensor,
diff --git a/CMakeLists.txt b/CMakeLists.txt
index c7ed1b2e..0d0d5454 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -2,102 +2,10 @@ cmake_minimum_required(VERSION 3.31)
project(OpenCDA LANGUAGES NONE)
-option(OPENCDA_BUILD_CUDA "Build OpenCOOD CUDA extensions" OFF)
-option(OPENCDA_BUILD_PROTOBUF "Generate OpenCDA protobuf Python modules" OFF)
+option(OPENCDA_BUILD_PROTOBUF "Generate OpenCDA protobuf Python modules" ON)
-if(NOT OPENCDA_BUILD_CUDA AND NOT OPENCDA_BUILD_PROTOBUF)
- message(FATAL_ERROR
- "No build component selected. Enable OPENCDA_BUILD_CUDA or "
- "OPENCDA_BUILD_PROTOBUF."
- )
+if(NOT OPENCDA_BUILD_PROTOBUF)
+ message(FATAL_ERROR "No OpenCDA build component selected")
endif()
-if(OPENCDA_BUILD_PROTOBUF)
- add_subdirectory(opencda/core/common/communication)
-endif()
-
-if(OPENCDA_BUILD_CUDA)
- if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES OR CMAKE_CUDA_ARCHITECTURES STREQUAL "")
- message(FATAL_ERROR
- "CMAKE_CUDA_ARCHITECTURES must be set explicitly when building "
- "the OpenCOOD CUDA extensions."
- )
- endif()
-
- set(OPENCDA_CUDA_ARCHITECTURES "${CMAKE_CUDA_ARCHITECTURES}")
- set(OPENCDA_TORCH_CUDA_ARCH_LIST)
- foreach(cuda_architecture IN LISTS OPENCDA_CUDA_ARCHITECTURES)
- if(NOT cuda_architecture MATCHES "^[0-9]+$")
- message(FATAL_ERROR
- "Unsupported CUDA architecture '${cuda_architecture}'. "
- "Use numeric values such as 86 or 75;86;89."
- )
- endif()
-
- math(EXPR cuda_architecture_major "${cuda_architecture} / 10")
- math(EXPR cuda_architecture_minor "${cuda_architecture} % 10")
- list(APPEND OPENCDA_TORCH_CUDA_ARCH_LIST
- "${cuda_architecture_major}.${cuda_architecture_minor}"
- )
- endforeach()
-
- enable_language(CXX)
- enable_language(CUDA)
-
- set(CMAKE_CXX_STANDARD 17)
- set(CMAKE_CXX_STANDARD_REQUIRED ON)
- set(CMAKE_CXX_EXTENSIONS OFF)
- set(CMAKE_CUDA_STANDARD 17)
- set(CMAKE_CUDA_STANDARD_REQUIRED ON)
- set(CMAKE_CUDA_EXTENSIONS OFF)
- set(CMAKE_POSITION_INDEPENDENT_CODE ON)
-
- find_package(Python3 3.12 REQUIRED COMPONENTS Interpreter Development.Module)
- find_package(CUDAToolkit REQUIRED)
-
- execute_process(
- COMMAND
- "${Python3_EXECUTABLE}" -c
- "import torch; print(torch.utils.cmake_prefix_path)"
- OUTPUT_VARIABLE TORCH_CMAKE_PREFIX_PATH
- OUTPUT_STRIP_TRAILING_WHITESPACE
- COMMAND_ERROR_IS_FATAL ANY
- )
- list(APPEND CMAKE_PREFIX_PATH "${TORCH_CMAKE_PREFIX_PATH}")
-
- # PyTorch's CMake package uses TORCH_CUDA_ARCH_LIST and otherwise adds a
- # broad architecture list that includes targets removed from recent CUDA
- # toolkits. Disable CMake's duplicate architecture flags and let PyTorch
- # emit the requested gencode flags.
- unset(CMAKE_CUDA_ARCHITECTURES CACHE)
- unset(CMAKE_CUDA_ARCHITECTURES)
- set(TORCH_CUDA_ARCH_LIST "${OPENCDA_TORCH_CUDA_ARCH_LIST}")
- message(STATUS
- "Building OpenCOOD CUDA extensions for ${TORCH_CUDA_ARCH_LIST}"
- )
- find_package(Torch REQUIRED)
-
- execute_process(
- COMMAND
- "${Python3_EXECUTABLE}" -c
- "from pathlib import Path; import torch; print(Path(torch.__file__).parent / 'lib')"
- OUTPUT_VARIABLE OPENCDA_DETECTED_TORCH_LIBRARY_DIR
- OUTPUT_STRIP_TRAILING_WHITESPACE
- COMMAND_ERROR_IS_FATAL ANY
- )
- set(
- OPENCDA_TORCH_RUNTIME_LIBRARY_DIR
- "${OPENCDA_DETECTED_TORCH_LIBRARY_DIR}"
- CACHE PATH
- "Runtime directory containing the PyTorch shared libraries"
- )
- find_library(
- OPENCDA_TORCH_PYTHON_LIBRARY
- NAMES torch_python
- PATHS "${OPENCDA_DETECTED_TORCH_LIBRARY_DIR}"
- NO_DEFAULT_PATH
- REQUIRED
- )
-
- add_subdirectory(OpenCOOD/opencood/pcdet_utils)
-endif()
+add_subdirectory(opencda/core/common/communication)
diff --git a/CMakePresets.json b/CMakePresets.json
index 6aa31715..eeab2b4a 100644
--- a/CMakePresets.json
+++ b/CMakePresets.json
@@ -3,10 +3,6 @@
{
"configurePreset": "protobuf",
"name": "protobuf"
- },
- {
- "configurePreset": "cuda",
- "name": "cuda"
}
],
"cmakeMinimumRequired": {
@@ -18,23 +14,11 @@
{
"binaryDir": "${sourceDir}/build/protobuf",
"cacheVariables": {
- "OPENCDA_BUILD_CUDA": "OFF",
"OPENCDA_BUILD_PROTOBUF": "ON"
},
"displayName": "OpenCDA protobuf modules",
"generator": "Ninja",
"name": "protobuf"
- },
- {
- "binaryDir": "${sourceDir}/build/cuda",
- "cacheVariables": {
- "CMAKE_BUILD_TYPE": "Release",
- "OPENCDA_BUILD_CUDA": "ON",
- "OPENCDA_BUILD_PROTOBUF": "OFF"
- },
- "displayName": "OpenCOOD CUDA extensions",
- "generator": "Ninja",
- "name": "cuda"
}
],
"version": 6
diff --git a/Dockerfile b/Dockerfile
index 8aea47ee..249771d0 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -14,6 +14,8 @@ ENV HOME=${HOME}
ENV PATH="/opt/venv/bin:${PATH}"
ENV XDG_RUNTIME_DIR=/tmp/runtime-${USER}
ENV OPENCDA_WORKSPACE=${HOME}/cavise/opencda
+ENV OPENCOOD_WORKSPACE=${HOME}/cavise/opencood
+ENV CAVISE_MODELS_ROOT=${HOME}/cavise/models
RUN apt-get update && \
apt-get install -y --no-install-recommends \
@@ -39,7 +41,7 @@ RUN if id -u ubuntu >/dev/null 2>&1; then userdel -r ubuntu; fi && \
chown -R ${UID}:${UID} ${XDG_RUNTIME_DIR} ${HOME}
-FROM runtime-base AS python-dependencies
+FROM runtime-base AS python-opencda-dependencies
RUN apt-get update && \
apt-get install -y --no-install-recommends python3-venv=3.12.* && \
@@ -57,6 +59,14 @@ RUN python3 -m venv /opt/venv && \
-r /tmp/opencda-requirements/requirements.txt
+FROM python-opencda-dependencies AS python-opencood-dependencies
+
+COPY opencood/requirements.txt /tmp/opencood-requirements/requirements.txt
+
+RUN /opt/venv/bin/python -m pip install --no-cache-dir \
+ -r /tmp/opencood-requirements/requirements.txt
+
+
FROM ubuntu:${UBUNTU_VERSION}@${UBUNTU_DIGEST} AS protobuf-builder
ARG PROTOC_VERSION=36.0
@@ -98,14 +108,13 @@ COPY opencda/opencda/core/common/communication/messages/ \
RUN cmake -S . -B /tmp/opencda-protobuf-build -G Ninja \
-DOPENCDA_BUILD_PROTOBUF=ON \
- -DOPENCDA_BUILD_CUDA=OFF \
&& \
cmake --build /tmp/opencda-protobuf-build && \
cmake --install /tmp/opencda-protobuf-build \
--prefix /opt/opencda-artifacts/protobuf
-FROM docker.io/nvidia/cuda:${CUDA_VERSION}-devel-ubuntu${UBUNTU_VERSION}@${CUDA_DEVEL_DIGEST} AS cuda-builder
+FROM docker.io/nvidia/cuda:${CUDA_VERSION}-devel-ubuntu${UBUNTU_VERSION}@${CUDA_DEVEL_DIGEST} AS opencood-cuda-builder
ARG CUDA_ARCHITECTURES=86
@@ -120,7 +129,7 @@ RUN apt-get update && \
&& \
rm -rf /var/lib/apt/lists/*
-COPY opencda/requirements-cuda.txt /tmp/requirements-cuda.txt
+COPY opencood/requirements-cuda.txt /tmp/requirements-cuda.txt
RUN python3 -m venv /opt/cuda-build-venv && \
python -m pip install --no-cache-dir \
@@ -128,22 +137,19 @@ RUN python3 -m venv /opt/cuda-build-venv && \
ninja==1.13.2 \
-r /tmp/requirements-cuda.txt
-WORKDIR /src/opencda
+WORKDIR /src/opencood
-COPY opencda/CMakeLists.txt ./
-COPY opencda/OpenCOOD/opencood/pcdet_utils/ \
- OpenCOOD/opencood/pcdet_utils/
+COPY opencood/CMakeLists.txt ./
+COPY opencood/opencood/pcdet_utils/ opencood/pcdet_utils/
-RUN cmake -S . -B /tmp/opencda-cuda-build -G Ninja \
- -DOPENCDA_BUILD_PROTOBUF=OFF \
- -DOPENCDA_BUILD_CUDA=ON \
+RUN cmake -S . -B /tmp/opencood-cuda-build -G Ninja \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_CUDA_ARCHITECTURES=${CUDA_ARCHITECTURES} \
- -DOPENCDA_TORCH_RUNTIME_LIBRARY_DIR=/opt/venv/lib/python3.12/site-packages/torch/lib \
+ -DOPENCOOD_TORCH_RUNTIME_LIBRARY_DIR=/opt/venv/lib/python3.12/site-packages/torch/lib \
&& \
- cmake --build /tmp/opencda-cuda-build && \
- cmake --install /tmp/opencda-cuda-build \
- --prefix /opt/opencda-artifacts/cuda
+ cmake --build /tmp/opencood-cuda-build && \
+ cmake --install /tmp/opencood-cuda-build \
+ --prefix /opt/opencood-artifacts/cuda
FROM runtime-base AS opencda-base
@@ -152,13 +158,12 @@ ARG USER=opencda
ARG UID=1000
ARG HOME=/home/${USER}
-COPY --from=python-dependencies /opt/venv /opt/venv
+COPY --from=python-opencda-dependencies /opt/venv /opt/venv
WORKDIR ${HOME}/cavise/opencda
COPY --chown=${UID}:${UID} opencda/opencda.py ./
COPY --chown=${UID}:${UID} opencda/opencda/ opencda/
-COPY --chown=${UID}:${UID} opencda/OpenCOOD/ OpenCOOD/
COPY --chown=${UID}:${UID} opencda/AIM/ AIM/
COPY opencda/docker/sync-build-artifacts.sh \
/usr/local/bin/opencda-sync-build-artifacts
@@ -184,19 +189,47 @@ COPY --from=protobuf-builder /opt/opencda-artifacts/protobuf \
/opt/opencda-artifacts/protobuf
-FROM opencda-base AS opencda-cuda
+FROM opencda-base AS opencda-coperception-base
+
+ARG USER=opencda
+ARG UID=1000
+
+USER 0
+
+RUN apt-get update && \
+ apt-get install -y --no-install-recommends git=1:2.43.* && \
+ rm -rf /var/lib/apt/lists/*
+
+COPY --from=python-opencood-dependencies /opt/venv /opt/venv
+COPY --chown=${UID}:${UID} opencood/ ${OPENCOOD_WORKSPACE}/
+COPY opencood/docker/sync-cuda-artifacts.sh \
+ /usr/local/bin/opencood-sync-cuda-artifacts
+
+RUN chmod 755 /usr/local/bin/opencood-sync-cuda-artifacts && \
+ python -m pip install --no-cache-dir --no-build-isolation --no-deps \
+ --editable ${OPENCOOD_WORKSPACE}
+
+USER ${USER}
+
+ENTRYPOINT ["/usr/local/bin/opencood-sync-cuda-artifacts", "/usr/local/bin/opencda-sync-build-artifacts"]
-ENV OPENCDA_NATIVE_COMPONENTS="cuda"
-COPY --from=cuda-builder /opt/opencda-artifacts/cuda \
- /opt/opencda-artifacts/cuda
+FROM opencda-coperception-base AS opencda-coperception
+ENV OPENCOOD_NATIVE_COMPONENTS=""
-FROM opencda-base AS opencda
-ENV OPENCDA_NATIVE_COMPONENTS="protobuf cuda"
+FROM opencda-coperception-base AS opencda-cuda
+
+ENV OPENCOOD_NATIVE_COMPONENTS="cuda"
+
+COPY --from=opencood-cuda-builder /opt/opencood-artifacts/cuda \
+ /opt/opencood-artifacts/cuda
+
+
+FROM opencda-cuda AS opencda
+
+ENV OPENCDA_NATIVE_COMPONENTS="protobuf"
COPY --from=protobuf-builder /opt/opencda-artifacts/protobuf \
/opt/opencda-artifacts/protobuf
-COPY --from=cuda-builder /opt/opencda-artifacts/cuda \
- /opt/opencda-artifacts/cuda
diff --git a/OpenCOOD/.devcontainer/ubuntu/Dockerfile b/OpenCOOD/.devcontainer/ubuntu/Dockerfile
deleted file mode 100644
index 9c5afbe4..00000000
--- a/OpenCOOD/.devcontainer/ubuntu/Dockerfile
+++ /dev/null
@@ -1,37 +0,0 @@
-# TODO: Check why opencda mounts to devcontainer instead of just OpenCOOD
-FROM docker.io/nvidia/cuda:12.8.1-cudnn-devel-ubuntu24.04@sha256:24c8e3581ea6330038b0d374920721983312627f8adbfcf390bdb4b399d280ed AS opencda
-SHELL [ "/bin/bash", "-c"]
-ARG USER=devcontainer
-ARG UID=1000
-ARG HOME=/home/${USER}
-
-ENV TORCH_CUDA_ARCH_LIST="8.9"
-ENV TORCH_NVCC_FLAGS="--allow-unsupported-compiler"
-ENV PYTHONPATH="/workspaces/opencda/OpenCOOD"
-
-RUN userdel -r ubuntu && useradd -l -m -u ${UID} -s /bin/bash ${USER} -d ${HOME}
-
-RUN apt-get update && \
- apt-get install -y --no-install-recommends \
- python3-pip=24.0+* \
- python3-dev=3.12.* \
- libgl1=1.7.* \
- libglib2.0-0=2.80.* \
- libsm6=2:1.2.* \
- libxext6=2:1.3.* \
- libxrender1=1:0.9.* \
- && \
- apt-get autoremove -y && \
- rm -rf /var/lib/apt/lists/*
-
-RUN ln -s /usr/bin/python3 /usr/local/bin/python
-
-USER ${USER}
-WORKDIR ${HOME}
-
-COPY requirements.txt requirements.txt
-
-RUN python3 -m pip install --no-cache-dir --break-system-packages --upgrade pip==26.0.1 && \
- python3 -m pip install --no-cache-dir --break-system-packages -r requirements.txt
-
-CMD ["/bin/bash"]
diff --git a/OpenCOOD/.devcontainer/ubuntu/devcontainer.json b/OpenCOOD/.devcontainer/ubuntu/devcontainer.json
deleted file mode 100644
index c5eb1c1a..00000000
--- a/OpenCOOD/.devcontainer/ubuntu/devcontainer.json
+++ /dev/null
@@ -1,14 +0,0 @@
-{
- "build": {
- "args": {
- "USER": "devcontainer"
- },
- "context": "${localWorkspaceFolder}",
- "dockerfile": "${localWorkspaceFolder}/.devcontainer/ubuntu/Dockerfile"
- },
- "containerUser": "devcontainer",
- "name": "OpenCOOD DevContainer",
- "runArgs": [
- "--shm-size=8g"
- ]
-}
diff --git a/OpenCOOD/.gitignore b/OpenCOOD/.gitignore
deleted file mode 100644
index 0ab0c974..00000000
--- a/OpenCOOD/.gitignore
+++ /dev/null
@@ -1,140 +0,0 @@
-# Byte-compiled / optimized / DLL files
-__pycache__/
-*.py[cod]
-*$py.class
-
-.DS_Store
-
-# C extensions
-*.so
-
-# Distribution / packaging
-.Python
-build/
-develop-eggs/
-dist/
-downloads/
-eggs/
-.eggs/
-lib/
-lib64/
-parts/
-sdist/
-var/
-wheels/
-pip-wheel-metadata/
-share/python-wheels/
-*.egg-info/
-.installed.cfg
-*.egg
-MANIFEST
-
-# PyInstaller
-# Usually these files are written by a python script from a template
-# before PyInstaller builds the exe, so as to inject date/other infos into it.
-*.manifest
-*.spec
-
-# Installer logs
-pip-log.txt
-pip-delete-this-directory.txt
-
-# Unit test / coverage reports
-htmlcov/
-.tox/
-.nox/
-.coverage
-.coverage.*
-.cache
-nosetests.xml
-coverage.xml
-*.cover
-*.py,cover
-.hypothesis/
-.pytest_cache/
-
-# Translations
-*.mo
-*.pot
-
-# Django stuff:
-*.log
-local_settings.py
-db.sqlite3
-db.sqlite3-journal
-
-# Flask stuff:
-instance/
-.webassets-cache
-
-# Scrapy stuff:
-.scrapy
-
-# Sphinx documentation
-docs/_build/
-
-# PyBuilder
-target/
-
-# Jupyter Notebook
-.ipynb_checkpoints
-
-# IPython
-profile_default/
-ipython_config.py
-
-# pyenv
-.python-version
-
-# pipenv
-# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
-# However, in case of collaboration, if having platform-specific dependencies or dependencies
-# having no cross-platform support, pipenv may install dependencies that don't work, or not
-# install all needed dependencies.
-#Pipfile.lock
-
-# PEP 582; used by e.g. github.com/David-OConnor/pyflow
-__pypackages__/
-
-# Celery stuff
-celerybeat-schedule
-celerybeat.pid
-
-# SageMath parsed files
-*.sage.py
-
-# Environments
-.env
-.venv
-env/
-venv/
-ENV/
-env.bak/
-venv.bak/
-
-# Spyder project settings
-.spyderproject
-.spyproject
-
-# Rope project settings
-.ropeproject
-
-# mkdocs documentation
-/site
-
-# mypy
-.mypy_cache/
-.dmypy.json
-dmypy.json
-
-# Pyre type checker
-.pyre/
-logs/
-*.c
-*.so
-.idea
-
-opv2v_data_dumping
-checkpoints
-DCOS.md
-spconv/
diff --git a/OpenCOOD/.readthedocs.yaml b/OpenCOOD/.readthedocs.yaml
deleted file mode 100644
index 7a3b5af8..00000000
--- a/OpenCOOD/.readthedocs.yaml
+++ /dev/null
@@ -1,10 +0,0 @@
-# .readthedocs.yaml
-# Read the Docs configuration file
-# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
-
-# Required
-version: 2
-
-# Build documentation in the docs/ directory with Sphinx
-sphinx:
- configuration: docs/conf.py
diff --git a/OpenCOOD/LICENSE b/OpenCOOD/LICENSE
deleted file mode 100644
index 758e8d95..00000000
--- a/OpenCOOD/LICENSE
+++ /dev/null
@@ -1,10 +0,0 @@
-
-Academic Software License: © 2021 UCLA Mobility Lab (“Institution”). Academic or nonprofit researchers are permitted to use this Software (as defined below) subject to Paragraphs 1-3:
-
-Institution hereby grants to you free of charge, so long as you are an academic or nonprofit researcher, a nonexclusive license under Institution’s copyright ownership interest in this software and any derivative works made by you thereof (collectively, the “Software”) to use, copy, and make derivative works of the Software solely for educational or academic research purposes, in all cases subject to the terms of this Academic Software License. Except as granted herein, all rights are reserved by Institution, including the right to pursue patent protection of the Software.
-
-Please note you are prohibited from further transferring the Software -- including any derivatives you make thereof -- to any person or entity. Failure by you to adhere to the requirements in Paragraphs 1 and 2 will result in immediate termination of the license granted to you pursuant to this Academic Software License effective as of the date you first used the Software.
-
-IN NO EVENT SHALL INSTITUTION BE LIABLE TO ANY ENTITY OR PERSON FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE, EVEN IF INSTITUTION HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INSTITUTION SPECIFICALLY DISCLAIMS ANY AND ALL WARRANTIES, EXPRESS AND IMPLIED, INCLUDING, BUT NOT LIMITED TO, ANY IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE IS PROVIDED “AS IS.” INSTITUTION HAS NO OBLIGATION TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS OF THIS SOFTWARE.
-
-Commercial entities: please contact the UCLA Mobility Lab at jiaqima@ucla.edu for licensing opportunities.
diff --git a/OpenCOOD/README.md b/OpenCOOD/README.md
deleted file mode 100644
index 66e89fa7..00000000
--- a/OpenCOOD/README.md
+++ /dev/null
@@ -1,277 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-## Contributing:
-We use Ruff as a linter and formatter. Before creating a pull request, check for linter warnings and then run the formatter.
-Linter:
-```bash
-ruff check
-```
-Formatter:
-```bash
-ruff format
-```
-
-## News:
-- 12/28/2022: OpenCOOD now support multi-gpu training.
-- 12/21/2022: V2XSet (ECCV2022) is supported by OpenCOOD now!
-- 12/16/2022: Both spconv 1.2.1 and spconv 2.x are supported!
-- 12/04/2022: The log replay tool for OPV2V is online now! With
-this toolbox, you can 100% replay all the events in the offline dataset and add/change any sensors/groundtruth you
-want to explore the tasks that the origin dataset do not support. Check [here](logreplay/README.md) to see more details.
-- 09/15/2022: So far OpenCOOD has supported several top conference papers, including ECCV,ICRA,CoRL,NeurIPS,WACV! The bottom of this project page lists the detailed information.
-## Features
-- Provide easy data API for multiple popular multi-agent perception dataset:
- - [x] [OPV2V [ICRA2022]](https://mobility-lab.seas.ucla.edu/opv2v/)
- - [x] [V2XSet [ECCV2022]](https://arxiv.org/pdf/2203.10638.pdf)
- - [ ] [DAIR-V2X [CVPR2022]](https://arxiv.org/abs/2204.05575)
- - [ ] [V2V4Real [CVPR2023 Highlight]](https://arxiv.org/abs/2303.07601)
-- Provide APIs to allow users use different sensor modalities
- - [x] LiDAR APIs
- - [ ] Camera APIs
- - [ ] Radar APIs
-- Provide multiple SOTA 3D detection backbone:
- - [X] [PointPillar](https://arxiv.org/abs/1812.05784)
- - [X] [Pixor](https://arxiv.org/abs/1902.06326)
- - [X] [VoxelNet](https://arxiv.org/abs/1711.06396)
- - [X] [SECOND](https://www.mdpi.com/1424-8220/18/10/3337)
-- Support multiple sparse convolution versions
- - [X] Spconv 1.2.1
- - [X] Spconv 2.x
-- Support SOTA multi-agent perception models:
- - [x] [Attentive Fusion [ICRA2022]](https://arxiv.org/abs/2109.07644)
- - [x] [Cooper [ICDCS]](https://arxiv.org/abs/1905.05265)
- - [x] [F-Cooper [SEC2019]](https://arxiv.org/abs/1909.06459)
- - [x] [V2VNet [ECCV2022]](https://arxiv.org/abs/2008.07519)
- - [x] [CoAlign (fusion only) [ICRA2023]](https://arxiv.org/abs/2211.07214)
- - [x] [FPV-RCNN [RAL2022]](https://arxiv.org/pdf/2109.11615.pdf)
- - [ ] [DiscoNet [NeurIPS2021]](https://arxiv.org/abs/2111.00643)
- - [x] [V2X-ViT [ECCV2022]](https://github.com/DerrickXuNu/v2x-vit)
- - [x] [CoBEVT [CoRL2022]](https://arxiv.org/abs/2207.02202)
- - [ ] [AdaFusion [WACV2023]](https://arxiv.org/abs/2208.00116)
- - [x] [Where2comm [NeurIPS2022]](https://arxiv.org/abs/2209.12836)
- - [x] [V2VAM [TIV2023]](https://arxiv.org/abs/2212.08273)
-
-- **Provide a convenient log replay toolbox for OPV2V dataset.** Check [here](logreplay/README.md) to see more details.
-
-## Data Downloading
-All the data can be downloaded from [UCLA BOX](https://ucla.app.box.com/v/UCLA-MobilityLab-OPV2V). If you have a good internet, you can directly
-download the complete large zip file such as `train.zip`. In case you suffer from downloading large files, we also split each data set into small chunks, which can be found
-in the directory ending with `_chunks`, such as `train_chunks`. After downloading, please run the following command to each set to merge those chunks together:
-```python
-cat train.zip.part* > train.zip
-unzip train.zip
-```
-
-## Installation
-Please refer to [data introduction](https://opencood.readthedocs.io/en/latest/md_files/data_intro.html)
-and [installation](https://opencood.readthedocs.io/en/latest/md_files/installation.html) guide to prepare
-data and install OpenCOOD. To see more details of OPV2V data, please check [our website.](https://mobility-lab.seas.ucla.edu/opv2v/)
-
-## Quick Start
-### Data sequence visualization
-To quickly visualize the LiDAR stream in the OPV2V dataset, first modify the `validate_dir`
-in your `opencood/hypes_yaml/visualization.yaml` to the opv2v data path on your local machine, e.g. `opv2v/validate`,
-and the run the following commond:
-```python
-cd ~/OpenCOOD
-python opencood/visualization/vis_data_sequence.py [--color_mode ${COLOR_RENDERING_MODE}]
-```
-Arguments Explanation:
-- `color_mode` : str type, indicating the lidar color rendering mode. You can choose from 'constant', 'intensity' or 'z-value'.
-
-
-### Train your model
-OpenCOOD uses yaml file to configure all the parameters for training. To train your own model
-from scratch or a continued checkpoint, run the following commonds:
-```python
-python opencood/tools/train.py --hypes_yaml ${CONFIG_FILE} [--model_dir ${CHECKPOINT_FOLDER} --half]
-```
-Arguments Explanation:
-- `hypes_yaml`: the path of the training configuration file, e.g. `opencood/hypes_yaml/second_early_fusion.yaml`, meaning you want to train
-an early fusion model which utilizes SECOND as the backbone. See [Tutorial 1: Config System](https://opencood.readthedocs.io/en/latest/md_files/config_tutorial.html) to learn more about the rules of the yaml files.
-- `model_dir` (optional) : the path of the checkpoints. This is used to fine-tune the trained models. When the `model_dir` is
-given, the trainer will discard the `hypes_yaml` and load the `config.yaml` in the checkpoint folder.
-- `half` (optional): If set, the model will be trained with half precision. It cannot be set with multi-gpu training togetger.
-
-To train on **multiple gpus**, run the following command:
-```
-CUDA_VISIBLE_DEVICES=0,1,2,3 python -m torch.distributed.launch --nproc_per_node=4 --use_env opencood/tools/train.py --hypes_yaml ${CONFIG_FILE} [--model_dir ${CHECKPOINT_FOLDER}]
-```
-
-
-### Test the model
-Before you run the following command, first make sure the `validation_dir` in config.yaml under your checkpoint folder
-refers to the testing dataset path, e.g. `opv2v_data_dumping/test`.
-
-```python
-python opencood/tools/inference.py --model_dir ${CHECKPOINT_FOLDER} --fusion_method ${FUSION_STRATEGY} [--show_vis] [--show_video_vis]
-```
-Arguments Explanation:
-- `model_dir`: the path to your saved model.
-- `fusion_method`: indicate the fusion strategy, currently support 'early', 'late', and 'intermediate'.
-- `show_vis`: whether to visualize the detection overlay with point cloud.
-- `show_video_vis` : the detection results will visualized in a video stream. It can NOT be set with `show_vis` at the same time.
-- `global_sort_detections`: whether to globally sort detections by confidence score. If set to True, it is the mainstream AP computing method, but would increase the tolerance for FP (False Positives). **OPV2V paper does not perform the global sort.** Please choose the consistent AP calculation method in your paper for fair comparison.
-
-The evaluation results will be dumped in the model directory.
-
-## Benchmark and model zoo
-### Results on OPV2V LiDAR-track (AP@0.7 for no-compression/ compression)
-
-| | Spconv Version| Backbone | Fusion Strategy | Bandwidth (Megabit),
before/after compression| Default Towns |Culver City| Download |
-|--------------------| -------|-------- | --------------- | --------------- | ------------- |-----------| -------- |
-| Naive Late | 1.2.1 | PointPillar | Late | **0.024**/**0.024** | 0.781/0.781 | 0.668/0.668 | [url](https://ucla.app.box.com/v/UCLA-MobilityLab-OPV2V/file/1621128604521) |
-| [Cooper](https://arxiv.org/abs/1905.05265) | 1.2.1 | PointPillar | Early | 7.68/7.68 | 0.800/x | 0.696/x | [url](https://ucla.app.box.com/v/UCLA-MobilityLab-OPV2V/file/1621122534978) |
-| [Attentive Fusion](https://arxiv.org/abs/2109.07644) | 1.2.1 | PointPillar | Intermediate | 126.8/1.98 | 0.815/0.810 | 0.735/0.731 | [url](https://ucla.app.box.com/v/UCLA-MobilityLab-OPV2V/file/1621110356814) |
-| [F-Cooper](https://arxiv.org/abs/1909.06459) | 1.2.1 | PointPillar | Intermediate | 72.08/1.12 | 0.790/0.788 | 0.728/0.726 | [url](https://ucla.app.box.com/v/UCLA-MobilityLab-OPV2V/file/1621123814293) |
-| [V2VNet](https://arxiv.org/abs/2008.07519) | 1.2.1 | PointPillar | Intermediate | 72.08/1.12 | 0.822/0.814 | 0.734/0.729 | [url](https://ucla.app.box.com/v/UCLA-MobilityLab-OPV2V/file/1621111444798) |
-| [CoAlign](https://arxiv.org/abs/2211.07214) | 1.2.1 | PointPillar | Intermediate | 72.08/2.24 | **0.833**/0.806 | 0.760/ **0.750** | [url](https://drive.google.com/file/d/1mUEI_Dh4tkG6-LG3QcZ05kK7oOGJzCGK/view?usp=sharing)
-| [FPV-RCNN](https://arxiv.org/abs/2109.11615) | 1.2.1 | PV-RCNN | Intermediate(2 stage) | 0.24/0.24 | 0.820/0.820 | 0.763/**0.763** | [url](https://ucla.app.box.com/v/UCLA-MobilityLab-OPV2V/file/1621109586667) |
-| [V2VAM](https://arxiv.org/pdf/2212.08273.pdf) | 1.2.1 | PointPillar | Intermediate | x/x | **0.860**/**0.860** | **0.813**/**0.791** | [url](https://drive.google.com/drive/folders/1gFY6Aedrbi3vLJdPZ3x9abPnfW5w_HKN?usp=sharing) |
-| [CoBEVT]( https://arxiv.org/pdf/2207.02202.pdf) | 2.0 | PointPillar | Intermediate | 72.08/1.12 | **0.861**/**0.836** | **0.773**/0.730 | [url](https://ucla.app.box.com/v/UCLA-MobilityLab-OPV2V/folder/280139625059) |
-| Naive Late | 1.2.1 | VoxelNet | Late | **0.024**/**0.024** | 0.738/0.738 | 0.588/0.588 | [url]() |
-| Cooper | 1.2.1 | VoxelNet | Early | 7.68/7.68 | 0.758/x | 0.677/x | [url](https://ucla.app.box.com/v/UCLA-MobilityLab-OPV2V/file/1621124922131) |
-| Attentive Fusion | 1.2.1 | VoxelNet | Intermediate | 576.71/1.12 | **0.864**/**0.852** | **0.775**/**0.746** | [url](https://ucla.app.box.com/v/UCLA-MobilityLab-OPV2V/file/1621111451952) |
-| Naive Late | 1.2.1 | SECOND | Late | **0.024**/**0.024** | 0.775/0.775 |0.682/0.682 | [url](https://ucla.app.box.com/v/UCLA-MobilityLab-OPV2V/file/1621113752957) |
-| Cooper | 1.2.1 | SECOND | Early | 7.68/7.68 | 0.813/x | 0.738/x | [url](https://drive.google.com/file/d/1Z9io1VNcU-urcRW8l0ogWCTVCB53mw4N/view?usp=sharing) |
-| Attentive | 1.2.1 | SECOND | Intermediate | 63.4/0.99 | **0.826**/**0.783** | **0.760**/**0.760** | [url](https://ucla.app.box.com/v/UCLA-MobilityLab-OPV2V/file/1621121166914) |
-| Naive Late | 1.2.1 | PIXOR | Late | **0.024**/**0.024** | 0.578/0.578 | 0.360/0.360 | [url]() |
-| Cooper | 1.2.1 | PIXOR | Early | 7.68/7.68 | 0.678/x | **0.558**/x | [url](https://ucla.app.box.com/v/UCLA-MobilityLab-OPV2V/file/1621127209949) |
-| Attentive | 1.2.1 | PIXOR | Intermediate | 313.75/1.22 | **0.687**/**0.612** | 0.546/**0.492** | [url]() |
-
-**Note**:
-* We suggest using **PointPillar** as the backbone when you are creating your method and try to compare with
-our benchmark, as we implement most of the SOTA methods with this backbone only.
-* We assume the transimssion rate is 27Mbp/s. Considering the frequency of LiDAR is 10Hz, the
-bandwidth requirement should be less than **2.7Mbp** to avoid severe delay.
-* A 'x' in the benchmark table represents the bandwidth requirement is too large, which
-can not be considered to employ in practice.
-
-
-### Results of BEV semantic segmentation on OPV2V camera-track (IoU)
-
-| | Backbone | Fusion Strategy | Vehicles| Road Surface |Lane| Download |
-|--------------------| -------- | --------------- | --------------- | ------------- |-----------| -------- |
-| No Fusion | CVT | No Fusion | 37.7 | 57.8 | 43.7 | [None]() |
-| Map Fusion | CVT | Late | 45.1 | 60.0 | 44.1 | [None]() |
-| [Attentive Fusion](https://arxiv.org/abs/2109.07644) | CVT | Intermediate | 51.9 |60.5 | 46.2 | [None]() |
-| [F-Cooper](https://arxiv.org/abs/1909.06459) | CVT | Intermediate |52.5 | 60.4 | 46.5 | [None]() |
-| [V2VNet](https://arxiv.org/abs/2008.07519) | CVT | Intermediate | 53.5 | 60.2 | 47.5 | [None]() |
-| [DiscoNet](https://arxiv.org/abs/2109.11615) | CVT | Intermediate | 52.9 | 60.7 | 45.8 | [None]() |
-| [FuseBEVT](https://arxiv.org/pdf/2207.02202.pdf) | CVT | Intermediate | 59.0 | 62.1 | 49.2 | [url]() |
-| [CoBEVT](https://arxiv.org/pdf/2207.02202.pdf) | SinBEVT | Intermediate | **60.4** | **63.0** | **53.0** | [url](https://ucla.app.box.com/v/UCLA-MobilityLab-OPV2V/folder/280139287054) |
-
-**Note**:
-To play with OPV2V camera data, please check here: https://github.com/DerrickXuNu/CoBEVT
-
-### Results of 3D Detection on V2XSet LiDAR-Track
-| Method | Spconv Version | Backbone | Perfect AP@0.5 | Perfect AP@0.7 | Noisy AP@0.5 | Noisy AP@0.7 | Download Link |
-|--------------|----------------|-------------|----------------|----------------|--------------|--------------|--------------------------------------------------------------------------|
-| No Fusion | 2.0 | PointPillar | 60.6 | 40.2 | 60.6 | 40.2 | |
-| Late Fusion | 2.0 | PointPillar | 72.7 | 62.0 | 54.9 | 30.7 | |
-| Early Fusion | 2.0 | PointPillar | 81.9 | 71.0 | 72.0 | 38.4 | |
-| [F-Cooper](https://arxiv.org/abs/1909.06459) | 2.0 | PointPillar | 84.0 | 68.0 | 71.5 | 46.9 | |
-| [Attentive Fusion](https://arxiv.org/abs/2109.07644) | 2.0 | PointPillar | 80.7 | 66.4 | 70.9 | 48.7 | |
-| [V2VNet](https://arxiv.org/abs/2008.07519) | 2.0 | PointPillar | 84.5 | 67.7 | 79.1 | 49.3 | |
-| [DiscoNet](https://arxiv.org/abs/2109.11615) | 2.0 | PointPillar | 84.4 | 69.5 | 79.8 | 54.1 |
-| [CoBEVT](https://arxiv.org/abs/2207.02202) | 2.0 | PointPillar | 84.9 | 66.0 | 81.1 | 54.3 | [url](https://ucla.app.box.com/v/UCLA-MobilityLab-V2XVIT/file/1622754822466) ||
-| [Where2Comm](https://arxiv.org/abs/2209.12836) | 2.0 | PointPillar | 85.5 | 65.4 | 82.0| 53.4 | [URL](https://ucla.app.box.com/v/UCLA-MobilityLab-OPV2V/file/1621122986293) ||
-| [V2X-ViT](https://arxiv.org/pdf/2203.10638.pdf) | 2.0 | PointPillar | **88.2** | **71.2** | **83.6** | **61.4** | [url](https://ucla.app.box.com/v/UCLA-MobilityLab-V2XVIT/folder/280421221519) |
-
-Important Notes for Training in V2XSet:
-1. When you train from scratch, please first set `async` and `loc_err` to false to train on perfect setting. Also, set `compression` to 0 at beginning.
-2. After the model on perfect setting converged, set `compression` to 32 (please change the config yaml in your trained model directory) and continue training on the perfect setting for another 1-2 epoches.
-3. Next, set `async` to true, `async_mode` to 'real', `async_overhead` to 200 or 300, `loc_err` to true, `xyz_std` to 0.2, `rpy_std` to 0.2, and then continue training your model on this noisy setting. Please note that you are free to change these noise setting during training to obtain better performance.
-4. Eventually, use the model fine-tuned on noisy setting as the test model for both perfect and noisy setting.
-
-## Tutorials
-We have a series of tutorials to help you understand OpenCOOD more. Please check the series of our [tutorials](https://opencood.readthedocs.io/en/latest/md_files/config_tutorial.html).
-
-
-## Citation
- If you are using our OpenCOOD framework or OPV2V dataset for your research, please cite the following paper:
- ```bibtex
-@inproceedings{xu2022opencood,
- author = {Runsheng Xu, Hao Xiang, Xin Xia, Xu Han, Jinlong Li, Jiaqi Ma},
- title = {OPV2V: An Open Benchmark Dataset and Fusion Pipeline for Perception with Vehicle-to-Vehicle Communication},
- booktitle = {2022 IEEE International Conference on Robotics and Automation (ICRA)},
- year = {2022}}
-```
-
-## Supported Projects
-OpenCOOD has supported several top conference papers in cooperative perception field.
-
-**V2V4Real: A large-scale real-world dataset for Vehicle-to-Vehicle Cooperative Perception**
- Runsheng Xu, Xin Xia, Jinlong Li, Hanzhao Li, Shuo Zhang, Zhengzhong Tu, Zonglin Meng, Hao Xiang, Xiaoyu Dong, Rui Song, Hongkai Yu, Bolei Zhou, Jiaqi Ma
-*CVPR 2023*
-[[Paper]]()[[Code]](https://github.com/ucla-mobility/V2V4Real)
-
-**Robust Collaborative 3D Object Detection in Presence of Pose Errors**
-Yifan Lu, Quanhao Li, Baoan Liu, Mehrdad Dianati, Chen Feng, Siheng Chen, Yanfeng Wang
-*ICRA 2023*
-[[Paper]](https://arxiv.org/abs/2211.07214)[[Code]](https://github.com/yifanlu0227/CoAlign)
-
-
-**Analyzing Infrastructure LiDAR Placement with Realistic LiDAR Simulation Library**
-Xinyu Cai, Wentao Jiang, Runsheng Xu, Wenquan Zhao, Jiaqi Ma, Si Liu, Yikang Li
-*ICRA 2023*
-[[Paper]](https://arxiv.org/abs/2211.15975)[[Code]](https://github.com/PJLab-ADG/LiDARSimLib-and-Placement-Evaluation)
-
-**Bridging the Domain Gap for Multi-Agent Perception**
-Runsheng Xu, Jinlong Li, Xiaoyu Dong, Hongkai Yu, Jiaqi Ma∗
-*ICRA 2023*
-[[Paper]](https://arxiv.org/pdf/2210.08451.pdf)[[Code]](https://github.com/DerrickXuNu/MPDA)
-
-**Model Agnostic Multi-agent Perception**
-Runsheng Xu, Weizhe Chen, Hao Xiang, Xin Xia, Lantao Liu, Jiaqi Ma∗
-*ICRA 2023*
-[[Paper]](https://arxiv.org/abs/2203.13168)[[Code]](https://github.com/DerrickXuNu/model_anostic)
-
-**Learning for Vehicle-to-Vehicle Cooperative Perception under Lossy Communication**
-Jinlong Li, Runsheng Xu, Xinyu Liu, Jin Ma, Zicheng Chi, Jiaqi Ma, Hongkai Yu
-*TIV 2023*
-[[Paper]](https://ieeexplore.ieee.org/document/10077757) [[Code]](https://github.com/jinlong17/V2VLC)
-
-
-**Where2comm: Communication-Efficient Collaborative Perception via Spatial Confidence Maps**
-Yue Hu, Shaoheng Fang, Zixing Lei, Yiqi Zhong, Siheng Chen
-*Neurips 2022*
-[[Paper]](https://arxiv.org/abs/2209.12836) [[Code]](https://github.774.gs/MediaBrain-SJTU/Where2comm)
-
-**Adaptive Feature Fusion for Cooperative Perception using LiDAR Point Clouds**
-Donghao Qiao, Farhana Zulkernine
-*WACV 2023*
-[[Paper]](https://arxiv.org/abs/2208.00116)
-
-
-**CoBEVT: Cooperative Bird's Eye View Semantic Segmentation with Sparse Transformers**
-Runsheng Xu*, Zhengzhong Tu*, Hao Xiang, Wei Shao, Bolei Zhou, Jiaqi Ma
-*CoRL2022*
-[[Paper]](https://arxiv.org/abs/2207.02202) [[Code]](https://github.com/DerrickXuNu/CoBEVT)
-
-**V2X-ViT: Vehicle-to-Everything Cooperative Perception with Vision Transformer**
-Runsheng Xu*, Hao Xiang*, Zhengzhong Tu*, Xin Xia, Ming-Hsuan Yang, Jiaqi Ma
-*ECCV2022*
-[[Paper]](https://arxiv.org/abs/2203.10638) [[Code]](https://github.com/DerrickXuNu/v2x-vit) [[Talk]](https://course.zhidx.com/c/MmQ1YWUyMzM1M2I3YzVlZjE1NzM=)
-
-**OPV2V: An Open Benchmark Dataset and Fusion Pipeline for Perception with Vehicle-to-Vehicle Communication**
-Runsheng Xu*, Hao Xiang*, Xin Xia, Jinlong Li, Jiaqi Ma
-*ICRA2022*
-[[Paper]](https://arxiv.org/abs/2109.07644) [[Website]](https://mobility-lab.seas.ucla.edu/opv2v/) [[Code]](https://github.com/DerrickXuNu/OpenCOOD)
diff --git a/OpenCOOD/docs/Makefile b/OpenCOOD/docs/Makefile
deleted file mode 100644
index d4bb2cbb..00000000
--- a/OpenCOOD/docs/Makefile
+++ /dev/null
@@ -1,20 +0,0 @@
-# Minimal makefile for Sphinx documentation
-#
-
-# You can set these variables from the command line, and also
-# from the environment for the first two.
-SPHINXOPTS ?=
-SPHINXBUILD ?= sphinx-build
-SOURCEDIR = .
-BUILDDIR = _build
-
-# Put it first so that "make" without argument is like "make help".
-help:
- @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
-
-.PHONY: help Makefile
-
-# Catch-all target: route all unknown targets to Sphinx using the new
-# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
-%: Makefile
- @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
diff --git a/OpenCOOD/docs/conf.py b/OpenCOOD/docs/conf.py
deleted file mode 100644
index 5401b2c4..00000000
--- a/OpenCOOD/docs/conf.py
+++ /dev/null
@@ -1,66 +0,0 @@
-# Configuration file for the Sphinx documentation builder.
-#
-# This file only contains a selection of the most common options. For a full
-# list see the documentation:
-# https://www.sphinx-doc.org/en/master/usage/configuration.html
-
-# -- Path setup --------------------------------------------------------------
-
-# If extensions (or modules to document with autodoc) are in another directory,
-# add these directories to sys.path here. If the directory is relative to the
-# documentation root, use os.path.abspath to make it absolute, like shown here.
-#
-import os
-import sys
-
-# sys.path.insert(0, os.path.abspath('.'))
-sys.path.insert(0, os.path.abspath("./../"))
-
-
-# -- Project information -----------------------------------------------------
-
-project = "OpenCOOD"
-copyright = "2021, UCLA Mobility Lab"
-author = "Runsheng Xu, Hao Xiang, Jiaqi Ma"
-
-# The full version, including alpha/beta/rc tags
-release = "0.1"
-
-
-# -- General configuration ---------------------------------------------------
-
-# Add any Sphinx extension module names here, as strings. They can be
-# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
-# ones.
-extensions = [
- "sphinx.ext.autodoc",
- "sphinx.ext.napoleon",
- "sphinx.ext.doctest",
- "sphinx.ext.intersphinx",
- "sphinx.ext.todo",
- "sphinx.ext.coverage",
- "sphinx.ext.mathjax",
- "recommonmark",
-]
-
-# Add any paths that contain templates here, relative to this directory.
-templates_path = ["_templates"]
-
-# List of patterns, relative to source directory, that match files and
-# directories to ignore when looking for source files.
-# This pattern also affects html_static_path and html_extra_path.
-exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
-
-todo_include_todos = False
-
-# -- Options for HTML output -------------------------------------------------
-
-# The theme to use for HTML and HTML Help pages. See the documentation for
-# a list of builtin themes.
-#
-html_theme = "sphinx_rtd_theme"
-
-# Add any paths that contain custom static files (such as style sheets) here,
-# relative to this directory. They are copied after the builtin static files,
-# so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ["_static"]
diff --git a/OpenCOOD/docs/index.rst b/OpenCOOD/docs/index.rst
deleted file mode 100644
index 64781a14..00000000
--- a/OpenCOOD/docs/index.rst
+++ /dev/null
@@ -1,57 +0,0 @@
-.. OpenCDA documentation master file, created by
- sphinx-quickstart on Fri Jul 2 11:48:53 2021.
- You can adapt this file completely to your liking, but it should at least
- contain the root `toctree` directive.
-
-Welcome to OpenCOOD's documentation!
-===================================
-OpenCOOD is an open-source cooperative detection framework for autonomous driving. It provides SOTA cooperative detection algorithms,
-convenient APIs for the large-scale simulated V2V perception dataset `OPV2V `_, and a set of useful tools for log replay.
-
-In collaboration with `OpenCDA `_ , OpenCOOD is mainly focus on offline cooperative perception training and testing. If you are
-interested in online cooperative perception and the corresponding closed-loop simulation test, OpenCDA will be the best tool.
-
-OpenCOOD is a work in progress. Many features on the roadmap are being continuously developed. We welcome your contribution and please visit our Github repo
-for the latest release.
-
- `OpenCOOD source code on Github `_
-
-.. toctree::
- :maxdepth: 1
- :caption: Getting Started
-
- md_files/data_intro.md
- md_files/installation.md
-
-.. toctree::
- :maxdepth: 1
- :caption: Tutorials
-
- md_files/config_tutorial.md
- md_files/data_annotation_tutorial.md
- md_files/logic_flow.md
-
-.. toctree::
- :maxdepth: 1
- :caption: Additional Information
-
- md_files/contributor.md
-
-**Citing OpenCOOD**\ :
-
-If you are using our OpenCOOD framework or codes for your development, please cite the following paper::
-
- @inproceedings{xu2022opencood,
- author = {Runsheng Xu, Hao Xiang, Xin Xia, Xu Han, Jinlong Li, Jiaqi Ma},
- title = {OPV2V: An Open Benchmark Dataset and Fusion Pipeline for Perception with Vehicle-to-Vehicle Communication},
- booktitle = {2022 IEEE International Conference on Robotics and Automation (ICRA)},
- year = {2022}}
-
-Also, under this LICENSE, OpenCOOD is for non-commercial research only. Researchers can modify the source code for their own research only.
-
-Indices and tables
-==================
-
-* :ref:`genindex`
-* :ref:`modindex`
-* :ref:`search`
diff --git a/OpenCOOD/docs/make.bat b/OpenCOOD/docs/make.bat
deleted file mode 100644
index 2119f510..00000000
--- a/OpenCOOD/docs/make.bat
+++ /dev/null
@@ -1,35 +0,0 @@
-@ECHO OFF
-
-pushd %~dp0
-
-REM Command file for Sphinx documentation
-
-if "%SPHINXBUILD%" == "" (
- set SPHINXBUILD=sphinx-build
-)
-set SOURCEDIR=.
-set BUILDDIR=_build
-
-if "%1" == "" goto help
-
-%SPHINXBUILD% >NUL 2>NUL
-if errorlevel 9009 (
- echo.
- echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
- echo.installed, then set the SPHINXBUILD environment variable to point
- echo.to the full path of the 'sphinx-build' executable. Alternatively you
- echo.may add the Sphinx directory to PATH.
- echo.
- echo.If you don't have Sphinx installed, grab it from
- echo.http://sphinx-doc.org/
- exit /b 1
-)
-
-%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
-goto end
-
-:help
-%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
-
-:end
-popd
diff --git a/OpenCOOD/docs/md_files/config_tutorial.md b/OpenCOOD/docs/md_files/config_tutorial.md
deleted file mode 100644
index 3071201d..00000000
--- a/OpenCOOD/docs/md_files/config_tutorial.md
+++ /dev/null
@@ -1,139 +0,0 @@
-## Tutorial 1: Learn about Config System
-
----
-We incorporate modular and inheritance design into the config system to enable users conveniently
-modify the model/training/inference parameters. Specifically, we use **yaml** files to configure all the
-important parameters.
-
-### Config File Location
-All the yaml files should be saved in `opencood/hypes_yaml`, and users should use the `load_yaml()` function in [`opencood/hypes_yaml/yaml_utils.py`](https://github.com/DerrickXuNu/OpenCOOD/blob/31ba16025da27ffe4e336f011290dfbc66f9a1f1/opencood/hypes_yaml/yaml_utils.py#L14) to load the parameters into a dictionary.
-
-### Config Name Style
-We follow the below style to name config yaml files.
-```python
-{backbone}_{fusion_strategy}.yaml
-```
-
-### A concrete example
-Now let's go through the `point_pillar_intermediate_fusion.yaml` as an example.
-```yaml
-name: point_pillar_intermediate_fusion # this parameter together with the current timestamp will define the name of the saved folder for the model.
-root_dir: "v2xset/train" # this is where the training data locate. It can be either opv2v/train or v2xset/train
-validate_dir: "v2xset/validate" # during training, it defines the validation folder. during testing, it defines the testing folder path.
-
-yaml_parser: "load_point_pillar_params" # we need specific loading functions for different backbones.
-train_params: # the common training parameters
- batch_size: &batch_size 2
- epoches: 60
- eval_freq: 1
- save_freq: 1
-
-wild_setting: # setting related to noise
- async: true
- async_mode: 'sim'
- async_overhead: 100
- backbone_delay: 10
- data_size: 1.06
- loc_err: true
- ryp_std: 0.2
- seed: 25
- transmission_speed: 27
- xyz_std: 0.2
-
-fusion:
- core_method: 'IntermediateFusionDataset' # LateFusionDataset, EarlyFusionDataset, and IntermediateFusionDataset are supported
- args:
- cur_ego_pose_flag: True
- # when the cur_ego_pose_flag is set to True, there is no time gap
- # between the time when the LiDAR data is captured by connected
- # agents and when the extracted features are received by
- # the ego vehicle, which is equal to implement STCM. When set to False,
- # STCM has to be used. To validate STCM, V2X-ViT will set this as False.
-
-# preprocess-related
-preprocess:
- # options: BasePreprocessor, SpVoxelPreprocessor, BevPreprocessor
- core_method: 'SpVoxelPreprocessor'
- args:
- voxel_size: &voxel_size [0.4, 0.4, 4] # the voxel resolution for PointPillar
- max_points_per_voxel: 32 # maximum points allowed in each voxel
- max_voxel_train: 32000 # the maximum voxel number during training
- max_voxel_test: 70000 # the maximum voxel number during testing
- # LiDAR point cloud cropping range
- cav_lidar_range: &cav_lidar [-140.8, -40, -3, 140.8, 40, 1]
-
-# data augmentation options.
-data_augment:
- - NAME: random_world_flip
- ALONG_AXIS_LIST: [ 'x' ]
-
- - NAME: random_world_rotation
- WORLD_ROT_ANGLE: [ -0.78539816, 0.78539816 ]
-
- - NAME: random_world_scaling
- WORLD_SCALE_RANGE: [ 0.95, 1.05 ]
-
-# post processing related.
-postprocess:
- core_method: 'VoxelPostprocessor' # VoxelPostprocessor and BevPostprocessor are supported
- anchor_args: # anchor generator parameters
- cav_lidar_range: *cav_lidar # the range is consistent with the lidar cropping range to generate the correct ancrhors
- l: 3.9 # the default length of the anchor
- w: 1.6 # the default width
- h: 1.56 # the default height
- r: [0, 90] # the yaw angles. 0, 90 meaning for each voxel, two anchors will be generated with 0 and 90 degree yaw angle
- feature_stride: 2 # the feature map is shrank twice compared the input voxel tensor
- num: &achor_num 2 # for each location in the feature map, 2 anchors will be generated
- target_args: # used to generate positive and negative samples for object detection
- pos_threshold: 0.6
- neg_threshold: 0.45
- score_threshold: 0.20
- order: 'hwl' # hwl or lwh
- max_num: 100 # maximum number of objects in a single frame. use this number to make sure different frames have the same dimension in the same batch
- nms_thresh: 0.15
-
-# model related
-model:
- core_method: point_pillar_opv2v # trainer will load the corresponding model python file with the same name
- args: # detailed parameters of the point pillar model
- voxel_size: *voxel_size
- lidar_range: *cav_lidar
- anchor_number: *achor_num
-
- pillar_vfe:
- use_norm: true
- with_distance: false
- use_absolute_xyz: true
- num_filters: [64]
- point_pillar_scatter:
- num_features: 64
-
- base_bev_backbone:
- layer_nums: [3, 5, 8]
- layer_strides: [2, 2, 2]
- num_filters: [64, 128, 256]
- upsample_strides: [1, 2, 4]
- num_upsample_filter: [128, 128, 128]
- compression: 0 # whether to compress the features before fusion to reduce the bandwidth
- backbone_fix: false # whether fix the pointpillar backbone weights during training.
- anchor_num: *achor_num
-
-loss: # loss function
- core_method: point_pillar_loss # trainer will load the loss function with the same name
- args:
- cls_weight: 1.0 # classification weights
- reg: 2.0 # regression weights
-
-optimizer: # optimzer setup
- core_method: Adam # the name has to exist in Pytorch optimizer library
- lr: 0.002
- args:
- eps: 1e-10
- weight_decay: 1e-4
-
-lr_scheduler: # learning rate schedular
- core_method: multistep #step, multistep and Exponential are supported
- gamma: 0.1
- step_size: [15, 30]
-
-```
diff --git a/OpenCOOD/docs/md_files/contributor.md b/OpenCOOD/docs/md_files/contributor.md
deleted file mode 100644
index 210915c8..00000000
--- a/OpenCOOD/docs/md_files/contributor.md
+++ /dev/null
@@ -1,18 +0,0 @@
-## About Us
-OpenCOOD is brought to you by [UCLA Mobility Lab](https://mobility-lab.seas.ucla.edu/).
-
-### Supervisor
-Dr. Jiaqi Ma (Associate Professor @ UCLA)
-- Lab Principal Investigator
-- [Linkedin](https://www.linkedin.com/in/jiaqi-ma-17037838/)
-- [Google Scholar](https://scholar.google.com/citations?user=S3cQz1AAAAAJ&hl=en)
-
-
-### Core Developer
-Runsheng Xu (Ph.D. Student @ UCLA):
-- [Homepage](https://derrickxunu.github.io/)
-- [Linkedin](https://www.linkedin.com/in/runsheng-xu/)
-
-Hao Xiang (Ph.D. Student @ UCLA):
-- [Homepage](https://xhwind.github.io/)
-- [Linkedin](https://www.linkedin.com/in/hao-xiang-42bb5a1b2/)
diff --git a/OpenCOOD/docs/md_files/data_annotation_tutorial.md b/OpenCOOD/docs/md_files/data_annotation_tutorial.md
deleted file mode 100644
index 1e021963..00000000
--- a/OpenCOOD/docs/md_files/data_annotation_tutorial.md
+++ /dev/null
@@ -1,93 +0,0 @@
-## Tutorial 2: Data Annotation Introduction
-
----
-We save all groundtruth annotations per agent per timestamp in the yaml files. For instance,
-`2021_08_24_21_29_28/4805/000069.yaml` refers to the data annotations with the perspective of te
-agent 4805 at timestamp 69 in the scenario database `2021_08_24_21_29_28`. Here we go through an example:
-
-```yaml
-camera0: # parameters for frontal camera
- cords: # the x,y,z,roll,yaw,pitch under CARLA map coordinate
- - 141.35067749023438
- - -388.642578125
- - 1.0410505533218384
- - 0.07589337974786758
- - 174.18048095703125
- - 0.20690691471099854
- extrinsic: # extrinsic matrix from camera to LiDAR
- - - 0.9999999999999999
- - -5.1230071481984265e-18
- - 9.322129061605055e-20
- - -2.999993025731527
- - - -2.5011383190939924e-18
- - 1.0
- - 1.1458579204685086e-19
- - -3.934422863949294e-06
- - - 2.7713237218713775e-20
- - 3.7310309839064755e-20
- - 1.0
- - 0.8999999040861146
- - - 0.0
- - 0.0
- - 0.0
- - 1.0
- intrinsic: # camera intrinsic matrix
- - - 335.639852470912
- - 0.0
- - 400.0
- - - 0.0
- - 335.639852470912
- - 300.0
- - - 0.0
- - 0.0
- - 1.0
-camera1: ... # params of right rear camera
-camera2: ... # params of left rear camera
-canera3: ... # params of back camera
-ego_speed: 18.13 # agent's current speed, km/h
-lidar_pose: # LiDAR pose under CARLA map coordinate system
-- 144.33
-- -388.94
-- 1.93
-- 0.078
-- 174.18
-- 0.21
-plan_trajectory: # agent's planning trajectory
-- - 140.
- - -388
- - 87
-predicted_ego_pos: # agent's localization (x,y,z,roll,yaw,pitch) gained from GPS
-- 143.78
-- -388.94
-- 0.036
-- 0.080
-- -185.95
-- 0.18
-true_ego_pos: # agent's true localization
-- 143.83
-- -388.89
-- 0.032
-- 0.075
-- 174.18
-- 0.21
-vehicles: # the surrounding vehicles that have at least one LiDAR point hit from the agent
- 4796: # the id of the vehicle (i.e. object)
- angle: # roll, yaw, pitch under CARLA map coordinate system
- - 0.096
- - -177.86
- - 0.197
- center: # the relative position from bounding box center to the frontal axis of this vehicle
- - 0.0004
- - 0.0005
- - 0.71
- extent: # half length, width and height of the vehicle in meter
- - 2.45
- - 1.06
- - 0.75
- location: # x, y ,z position of the center in the frontal axis of the vehicle under CARLA map coordinate system
- - 158.55
- - -385.75
- - 0.032
- speed: 19.47 # vehicle's speed
- 4880: ...
-```
diff --git a/OpenCOOD/docs/md_files/data_intro.md b/OpenCOOD/docs/md_files/data_intro.md
deleted file mode 100644
index ca333eea..00000000
--- a/OpenCOOD/docs/md_files/data_intro.md
+++ /dev/null
@@ -1,143 +0,0 @@
-# Data Preparation
-
----
-OpenCOOD support both **OPV2V (ICRA2022)** and **V2XSet (ECCV2022)** dataset.
-
-## OPV2V
-All the data can be downloaded from [google drive](https://drive.google.com/drive/folders/1dkDeHlwOVbmgXcDazZvO6TFEZ6V_7WUu). If you have a good internet, you can directly
-download the complete large zip file such as `train.zip`. In case you suffer from downloading large fiels, we also split each data set into small chunks, which can be found
-in the directory ending with `_chunks`, such as `train_chunks`. After downloading, please run the following command to each set to merge those chunks together:
-```python
-cat train.zip.part* > train.zip
-unzip train.zip
-```
-After downloading is finished, please make the file structured as following:
-
-```sh
-OpenCOOD # root of your OpenCOOD
-├── opv2v_data_dumping # the downloaded opv2v data
-│ ├── train
-│ ├── validate
-│ ├── test
-│ ├── test_culvercity
-├── opencood # the core codebase
-
-```
-
-
-OPV2V data is structured as following:
-
-```sh
-opv2v_data_dumping
-├── train # data for training
-│ ├── 2021_08_22_21_41_24 # scenario folder
-│ ├── data_protocol.yaml # the simulation parameters used to collect the data in Carla
-│ └── 1732 # The connected automated vehicle's id
-│ └── 00000.pcd - 00700.pcd # the point clouds data from timestamp 0 to 700
-│ ├── 00000.yaml - 00700.yaml # corresponding metadata for each timestamp
-│ ├── 00000_camera0.png - 00700_camera0.png # frontal camera images
-│ ├── 00000_camera1.png - 00700_camera1.png # right rear camera images
-│ ├── 00000_camera2.png - 00700_camera2.png # left rear camera images
-│ └── 00000_camera3.png - 00700_camera3.png # back camera images
-├── validate
-├── test
-├── test_culvercity
-```
-
-### 1. Data Split
-OPV2V dataset can be divided into 4 different folders: `train`, `validation`, `test`, and `test_culvercity`
-- `train`: contains all training data
-- `validate`: used for validation during training
-- `test`: test set for calrla default towns
-- `test_culvercity`: test set for the digital town of Culver City, Los Angeles
-
-### 2. Scenario Database
-OPV2V has 73 scenarios in total, where each of them contains data stream from different agents across different timestamps.
-Each scenario is named by the time it was gathered, e.g., `2021_08_22_21_41_24`.
-
-### 3. Agent Contents
-Under each scenario folder, the data of every intelligent agent~(i.e. connected automated vehicle) appearing in the current scenario is saved in different folders. Each folder is named by the agent's unique id, e.g., 1732.
-
-In each agent folder, data across different timestamps will be saved. Those timestamps are represented by five digits integers
-as the prefix of the filenames (e.g., 00700.pcd). There are three types of files inside the agent folders: LiDAR point clouds (`.pcd` files), camera images (`.png` files), and metadata (`.yaml` files).
-
-#### 3.1 Lidar point cloud
-The LiDAR data is saved with Open3d package and has a postfix ".pcd" in the name.
-
-#### 3.2 Camera images
-Each CAV is equipped with 4 RGB cameras (check https://mobility-lab.seas.ucla.edu/opv2v/ to see the mounted positions of these cameras) to capture the 360 degree of view of the surrounding scene.`camera0`, `camera1`, `camera2`, and `camera3` represent the front, right rear, left rear, and back cameras respectively.
-
-#### 3.3 Data Annotation
-All the metadata is saved in yaml files. It records the following important information at the current timestamp:
-- **ego information**: Current ego pose with and without GPS noise under Carla world coordinates, ego speed in km/h, the LiDAR pose, and future planning trajectories.
-- **calibration**: The intrinsic matrix and extrinsic matrix from each camera to the LiDAR sensor.
-- **objects annotation**: The pose and velocity of each surrounding human driving vehicle that has at least one point hit by the agent's LiDAR sensor. See [data annotation section](data_annotation_tutorial.md) for more details.
-
-### 4. Data Collection Protocol
-Besides agent contents, every scenario database also has a yaml file named `data_protocol.yaml`.
-This yaml file records the simulation configuration to collect the current scenario. It is used to log replay
-the data and enable users to add/modify sensors for new tasks without changing the original events.
-
-To help users collect customized data in CARLA (e.g., different sensor configurations and modalities) with similar data format and structure, we have released our OPV2V data collection code in the `feature/data_collection` branch of [OpenCDA](https://github.com/ucla-mobility/OpenCDA/tree/feature/data_collection). Users can refer to its [documentation](https://opencda-documentation.readthedocs.io/en/latest/md_files/introduction.html) for detailed instructions.
-
----
-## V2XSet
-The data can be found from [google url](https://drive.google.com/drive/folders/1r5sPiBEvo8Xby-nMaWUTnJIPK6WhY1B6?usp=sharing). Since the data for train/validate/test
-is very large, we split each data set into small chunks, which can be found in the directory ending with `_chunks`, such as `train_chunks`. After downloading, please run the following command to each set to merge those chunks together:
-```
-cat train.zip.part* > train.zip
-unzip train.zip
-```
-
-### 1. Structure
-After downloading is finished, please make the file structured as following:
-
-```sh
-OpenCOOD # root of opencood
-├── v2xset # the downloaded v2xset data
-│ ├── train
-│ ├── validate
-│ ├── test
-├── opencood # the core codebase
-```
-
-### 2. Label format
-V2XSet's data label format is nearly the same with OPV2V, except:
-- OPV2V only has vehicles while V2XSet has infrastructure sensors
-- All the infrastructure sensors' folder is named with negative integer, e.g. `-1`
-
----
-## Adding Noise to OPV2V and V2XSet
-
-Cooperative perception faces the challenge of GPS error and communication delay. Our OpenCOOD allows users
-to add realistic GPS error and communication delay to the dataset.
-
-To add noise to both OPV2V and V2XSet, just add the following parameters to the model yaml file:
-```
-wild_setting: # setting related to noise
- async: true
- async_mode: 'sim'
- async_overhead: 100
- backbone_delay: 10
- data_size: 1.06
- loc_err: true
- ryp_std: 0.2
- seed: 25
- transmission_speed: 27
- xyz_std: 0.2
-```
-`async`: whether add communication delay.
-`aysnc_mode`: sim or real mode. In sim mode, the delay is a constant while in real mode, the delay has a uniform distribution.
-The major experiment in the paper used sim mode whereas the 'Effects of transmission size' study used real
- mode.
-`async_overhead`: the communication delay in ms. In sim mode, it represents a constant number. In real mode,
-the systematic async will be a random number from 0 to `aysnc_overhead`.
-`backbone_delay`: an estimate of backbone computation time. Only useful in real mode.
-`data_size`: transmission data size in Mb. Only used in real mode.
-`transmission_speed`: data transmitting speed during communication. By default 27 Mb/s. Only used in real mode.
-`loc_err`: whether to add localization error.
-`xyz_std`: the standard deviation of positional GPS error.
-`ryp_std`: the standard deviation of angular GPS error.
-`seed`: random seed for noise simulation. please keep it as 25 during testing .
-
-To see a complete example, refer to [configuration tutorial section](config_tutorial.md)
diff --git a/OpenCOOD/docs/md_files/data_pipeline.png b/OpenCOOD/docs/md_files/data_pipeline.png
deleted file mode 100644
index de5a296f..00000000
Binary files a/OpenCOOD/docs/md_files/data_pipeline.png and /dev/null differ
diff --git a/OpenCOOD/docs/md_files/installation.md b/OpenCOOD/docs/md_files/installation.md
deleted file mode 100644
index 7b6a0227..00000000
--- a/OpenCOOD/docs/md_files/installation.md
+++ /dev/null
@@ -1,66 +0,0 @@
-# Installation
-
-* [__System/Hardware Requirements__](#requirements)
-* [__Installation__](#installation)
- * [__1. Dependency Installation__](#1-dependency-installation)
- * [__2. Install Pytorch__](#2-pytorch-installation-18)
- * [__3. Install Spconv__](#3-spconv-121-requred)
-
-
-
-
----
-## System/Hardware Requirements
-To get started, the following requirements should be fulfilled.
-* __System requirements.__ OpenCOOD is tested under Ubuntu 18.04
-* __Adequate GPU.__ A minimum of 6GB gpu is recommended.
-* __Disk Space.__ Estimate 100GB of space is recommended for data downoading.
-* __Python__ Python3.7 is required.
-
-
----
-## Installation
-### 1. Dependency Installation
-First, download OpenCOOD github to your local folder if you haven't done it yet.
-```sh
-git clone https://github.com/DerrickXuNu/OpenCOOD.git
-cd OpenCOOD
-```
-Next we create a conda environment and install the requirements.
-
-```sh
-conda env create -f environment.yml
-conda activate opencood
-python setup.py develop
-```
-
-If conda install failed, install through pip
-```sh
-pip install -r requirements.txt
-```
-
-### 2. Pytorch Installation (>=1.8, tested on 1.8-1.12.0)
-Go to https://pytorch.org/ to install pytorch cuda version.
-
-### 3. Spconv (1.2.1 or 2.x)
-OpenCOOD support both spconv 1.2.1 and 2.x to generate voxel features.
-
-To install spconv 1.2.1, please follow the guide in https://github.com/traveller59/spconv/tree/v1.2.1.
-
-To install spconv 2.x, please run the following commands (if you are using cuda 11.3):
-```python
-pip install spconv-cu113
-```
-#### Tips for installing spconv 1.2.1:
-1. make sure your cmake version >= 3.13.2
-2. CUDNN and CUDA runtime library (use `nvcc --version` to check) needs to be installed on your machine.
-
-
-
-### 4. Dependencies for FPV-RCNN (optional)
-The OpenCOOD CUDA extensions used by FPV-RCNN are built by the OpenCDA
-Docker CUDA builder. They are copied into the runtime image and synchronized
-into the source tree when the container starts.
-
-Rebuild the OpenCDA image after changing any source under
-`opencood/pcdet_utils`.
diff --git a/OpenCOOD/docs/md_files/lidar_benchmark.md b/OpenCOOD/docs/md_files/lidar_benchmark.md
deleted file mode 100644
index 12be8ca7..00000000
--- a/OpenCOOD/docs/md_files/lidar_benchmark.md
+++ /dev/null
@@ -1,29 +0,0 @@
-## Benchmark: 3D LiDAR Detection
-
----
-### Results on OPV2V dataset (AP@0.7 for no-compression/ compression)
-
-| | Backbone | Fusion Strategy | Bandwidth (Megabit),
before/after compression| Default Towns |Culver City| Download |
-|--------------------| -------- | --------------- | --------------- | ------------- |-----------| -------- |
-| Naive Late | PointPillar | Late | **0.024**/**0.024** | 0.781/0.781 | 0.668/0.668 | [url](https://drive.google.com/file/d/1WTKooW6k0exLqoIE5Czqy6ptycYlgKZz/view?usp=sharing) |
-| [Cooper](https://arxiv.org/abs/1905.05265) | PointPillar | Early | 7.68/7.68 | 0.800/x | 0.696/x | [url](https://drive.google.com/file/d/1N1p6syxGSKD18ELgtBQoSuUzR8tX1JeE/view?usp=sharing) |
-| [Attentive Fusion](https://arxiv.org/abs/2109.07644) | PointPillar | Intermediate | 126.8/1.98 | **0.815**/**0.810** | **0.735**/**0.731** | [url](https://drive.google.com/file/d/1u4w13SDzdGq6Irh2PHxT-qIlNXRT3z6Z/view?usp=sharing) |
-| [F-Cooper](https://arxiv.org/abs/1909.06459) | PointPillar | Intermediate | 72.08/1.12 | 0.790/0.788 | 0.728/0.726 | [url](https://drive.google.com/file/d/1CjXu9Y2ZTzJA6Oo3hnqFhbTqBVKq3mQb/view?usp=sharing) |
-| Naive Late | VoxelNet | Late | **0.024**/**0.024** | 0.738/0.738 | 0.588/0.588 | [url]() |
-| Cooper | VoxelNet | Early | 7.68/7.68 | 0.758/x | 0.677/x | [url](https://drive.google.com/file/d/14WD7iLLyyCJJ3lApbYYdr5KOUM1ACnve/view?usp=sharing) |
-| Attentive Fusion | VoxelNet | Intermediate | 576.71/1.12 | **0.864**/**0.852** | **0.775**/**0.746** | [url](https://drive.google.com/file/d/16q8CfcB8dS4EVhJMvvEfn0gM2ynxZB3E/view?usp=sharing) |
-| Naive Late | SECOND | Late | **0.024**/**0.024** | 0.775/0.775 |0.682/0.682 | [url](https://drive.google.com/file/d/1VG_FKe1mKagPVGXH7UGHpyaM5q3cxtD8/view?usp=sharing) |
-| Cooper | SECOND | Early | 7.68/7.68 | 0.813/x | 0.738/x | [url](https://drive.google.com/file/d/1Z9io1VNcU-urcRW8l0ogWCTVCB53mw4N/view?usp=sharing) |
-| Attentive | SECOND | Intermediate | 63.4/0.99 | **0.826**/**0.783** | **0.760**/**0.760** | [url](https://drive.google.com/file/d/1zEB8EyZ0X-WQykHFOM0pVwI8jXunRz1Z/view?usp=sharing) |
-| Naive Late | PIXOR | Late | **0.024**/**0.024** | 0.578/0.578 | 0.360/0.360 | [url]() |
-| Cooper | PIXOR | Early | 7.68/7.68 | 0.678/x | **0.558**/x | [url](https://drive.google.com/file/d/1ZDLjtizZCuV6D92LloEPKRIw-LqxfE1j/view?usp=sharing) |
-| Attentive | PIXOR | Intermediate | 313.75/1.22 | **0.687**/**0.612** | 0.546/**0.492** | [url]() |
-
-
-**Note**:
-* We suggest using **PointPillar** as the backbone when you are creating your method and try to compare with
-our benchmark, as we implement most of the SOTA methods with this backbone only.
-* We assume the transimssion rate is 27Mbp/s. Considering the frequency of LiDAR is 10Hz, the
-bandwidth requirement should be less than **2.7Mbp** to avoid severe delay.
-* A 'x' in the benchmark table represents the bandwidth requirement is too large, which
-can not be considered to employ in practice.
diff --git a/OpenCOOD/docs/md_files/logic_flow.md b/OpenCOOD/docs/md_files/logic_flow.md
deleted file mode 100644
index 440d235a..00000000
--- a/OpenCOOD/docs/md_files/logic_flow.md
+++ /dev/null
@@ -1,85 +0,0 @@
-## Tutorial 3: OpenCOOD Pipeline
-
----
-
-
-In this tutorial, we will go through the pipeline of OpenCOOD from loading data to generate structure predictions.
-As the figure shown above, there are five important modules: **configuration system**, **data loading**, **preprocessor**, **postprocessor**, and **model**.
-
-### Configuration System
-
----
-The configuration system will read the yaml file defined by users to load all parameters for data I/O, training schema, and model. The configuration will be saved in a dictionary. Please refer to [tutorial 1](config_tutorial.md) for more details.
-
-### Data Loading
-
----
-As different fusion strategies can lead to different data processing (e.g., early fusion aggregates all raw
-LiDAR point clouds whereas intermediate fusion aggregates intermediate deep features), we have three
-distinct PyTorch dataset class: `EarlyFusionDataset`, `LateFusionDataset`, and `IntermediateFusionDataset`.
-These three classes share the same interface and inherit from `BaseDataset`.
-
-#### BaseDataset
-There are two major tasks for `BaseDataset`.
-- First, given the path to the dataset, `BaseDataset` will construct a scenario database (dictionary format). The scenarior database is indexed by scenarior id, agent id and timestamp and returns file pathes for various sensors and saved yaml data. Here is an example of how the database structured:
-```yaml
-
-{scenario_id :
- {agent_1 :
- {timestamp1 :
- {yaml: path (str),
- lidar: path (str),
- cameras: alist of path to 4 dfferent cameras}}}}
-
-```
-- Second, during the runtime, given an index from PyTorch dataloader, `BaseDataset` will retrieve the corresponding path information from scenario database, load the data from disk and return the raw format data:
-```yaml
-{cav_id :
- {ego : True or False, # whether this CAV is regarded as ego
- params: a dictionary, # contains all metadata of this CAV and annotations for surrounding objects
- lidar_np: a (n, 4) numpy array for the raw point cloud,
- camera_rgbs: a list of rgb images from 4 cameras on the CAV.
- }
-}
-```
-
-#### EarlyFusion, LateFusion, IntermediateFusion dataset
-All of these 3 classes will inherit the `BaseDataset` to first construct a scenario database. During runtime, they will first call the `__get_item__` function from `BaseDataset` to get the raw format data. Next, they will do:
-
-1. Project all objects' bounding box coordinates from Carla Map global reference to the ego vehicle's reference. Note that for late fusion, the CAV project to its own coordinate frame instead of the ego vehicle.
-2. For early fusion and intermediate fusion, all agents will project their LiDAR point clouds to ego vehicle's coordinate frame.
-3. Call the corresponding `PreProcessor` to convert the raw point clouds to voxel/bev features.
-4. Call the corresponding `PostProcessor` to convert the objects annotation to the correct format for the loss computing.
-5. Structure all the information in a dictionary. The processed format will be:
-```yaml
-{ agent_id:
- { 'object_bbx_center': numpy array, # the bounding boxes of all objects with padding
- 'object_bbx_mask': numpy array, # indicate the valid index for object_bbx_center
- 'object_ids': a list of integer, # objects
- 'anchor_box': numpy array, # pre-defined anchor box for detection
- 'processed_lidar': numpy array, # voxel/bev features generated from preprocessor
- 'label_dict': numpy array, # the groundtruth that can be directly used for computing loss during training
- }
-}
-```
-Notice here we just mention the general process and data format, these 3 classes have some differences in details.
-For example, early fusion and intermediate fusion only have a single `agent_id` element -- `ego`, while late fusion
-has multiple `agent_id`. Please read the code for more details.
-
-#### Collate Function
-As our data format is customized, all 3 dataset classes have their own `collate_fn` for the PyTorch dataloader. The output of the collate function will be a similar dictionary as the one mentioned above except that all values are now in torch.Tensor format.
-
-### Preprocessor and Postprocessor
-
----
-The preprocessor is mainly used to convert the raw point clouds to voxels or 2D bev features (refer to PIXOR [paper](https://arxiv.org/abs/1902.06326)).
-
-The postprocessor has three purposes:
-- Convert the bounding boxes of the objects to the correct format so that the models can use to compute the loss.
-- Generate the predefined anchor box if needed.
-- Convert the output of the models to the correct format for evaluation and visualization.
-
-### Models
-
----
-All the models' architecture designs are saved in `opencood/models`. For early fusion and late fusion, each model file only have the backbone codes. For intermediate fusion, the model files have both backbone and fusion net codes.
diff --git a/OpenCOOD/docs/requirements.txt b/OpenCOOD/docs/requirements.txt
deleted file mode 100644
index 31071e3d..00000000
--- a/OpenCOOD/docs/requirements.txt
+++ /dev/null
@@ -1,6 +0,0 @@
-matplotlib==3.4.2
-numpy
-open3d
-opencv-python
-shapely
-tensorboardX
diff --git a/OpenCOOD/environment.yml b/OpenCOOD/environment.yml
deleted file mode 100644
index 6e8246b5..00000000
--- a/OpenCOOD/environment.yml
+++ /dev/null
@@ -1,17 +0,0 @@
-name: opencood
-channels:
- - defaults
-dependencies:
- - pip==21.1.2
- - python==3.7.11
- - pip:
- - matplotlib==3.4.2
- - numpy
- - open3d
- - opencv-python==4.5.5.62
- - pygame
- - tensorboardX
- - shapely==2.0.0
- - einops
- - cumm
- - timm
diff --git a/OpenCOOD/images/camera_demo.gif b/OpenCOOD/images/camera_demo.gif
deleted file mode 100644
index 2107af46..00000000
Binary files a/OpenCOOD/images/camera_demo.gif and /dev/null differ
diff --git a/OpenCOOD/images/demo1.gif b/OpenCOOD/images/demo1.gif
deleted file mode 100644
index b18b949b..00000000
Binary files a/OpenCOOD/images/demo1.gif and /dev/null differ
diff --git a/OpenCOOD/images/logreplay.gif b/OpenCOOD/images/logreplay.gif
deleted file mode 100644
index bacb3046..00000000
Binary files a/OpenCOOD/images/logreplay.gif and /dev/null differ
diff --git a/OpenCOOD/images/opencood.png b/OpenCOOD/images/opencood.png
deleted file mode 100644
index 6b9e2bd5..00000000
Binary files a/OpenCOOD/images/opencood.png and /dev/null differ
diff --git a/OpenCOOD/images/pytorch-logo-dark.png b/OpenCOOD/images/pytorch-logo-dark.png
deleted file mode 100644
index b7a1ceb9..00000000
Binary files a/OpenCOOD/images/pytorch-logo-dark.png and /dev/null differ
diff --git a/OpenCOOD/logreplay/README.md b/OpenCOOD/logreplay/README.md
deleted file mode 100644
index a3fe4796..00000000
--- a/OpenCOOD/logreplay/README.md
+++ /dev/null
@@ -1,42 +0,0 @@
-# CARLA Log Replay Tool for Cooperative Perception Dataset
-
-
-
-This repository provides a toolbox to replay simulated cooperative perception dataset in CARLA. With
-this toolbox, you can 100% replay all the events in the offline dataset and add/change any sensors/groundtruth you
-want to explore the tasks that the origin dataset do not support.
-
-## Features
-- Support replaying multiple cooperative perception dataset in CARLA
- - [x] [OPV2V [ICRA2022]](https://mobility-lab.seas.ucla.edu/opv2v/)
- - [ ] [V2XSet [ECCV2022]]()
-- Help you add new sensors to the agents to have additional sensor data, including but not limited to:
- - Depth sensor
- - Event camera
- - Semantic LiDAR
- - Radar
-
-- Dump new groundtruth that the origin dataset does not have, including but not limited to:
- - HDMap ([CoBEVT](https://arxiv.org/abs/2207.02202) (CoRL2022) used this groundtruth )
- - Trajectory for each vehicle
-- Change the number of CAVs! The original OPV2V data has fixed CAVs, but with this tool, you can
-assign any vehicle to a CAV equipped with sensors.
-
-## How to use
-### Step0: Install CARLA 0.9.12 and CARLA API to your python environment
-Check the CARLA official website: https://carla.readthedocs.io/en/latest/start_quickstart/
-### Step1: Yaml file configuration
-Please change the parameters in `logreplay/hypes_yaml/replay.yaml` to configure your replay.
-We have provided very detailed explanation for each line in the yaml.
-### Step2: add new sensors if necessary
-We have by default provide Semantic Lidar and BEV semantic camera sensors in the yaml file and codebase.
-If you want to add some new sensors, please add your sensor under `logreplay/sensors/` and refer to the `semantic_lidar`
-or `bev_semantic_camera.py` to write your own sensor class.
-### Step3: Run CARLA
-Start your CARLA simulator first. Right now we only support version 0.9.12
-### Step4: run the log replay tool
-Execute the following command:
-```python
-cd logreplay
-python scenario/scenarios_manager.py
-```
diff --git a/OpenCOOD/logreplay/__init__.py b/OpenCOOD/logreplay/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/OpenCOOD/logreplay/assets/__init__.py b/OpenCOOD/logreplay/assets/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/OpenCOOD/logreplay/assets/presave_lib.py b/OpenCOOD/logreplay/assets/presave_lib.py
deleted file mode 100644
index 1da9a501..00000000
--- a/OpenCOOD/logreplay/assets/presave_lib.py
+++ /dev/null
@@ -1,298 +0,0 @@
-BLUE_PRINT_LIB = {
- "vehicle.audi.a2": {
- "location": [1.1874361216523965e-10, 0.00033203110797330737, 0.7818066477775574],
- "extent": [1.852684736251831, 0.8943392634391785, 0.7745251059532166],
- "num_wheel": 4,
- },
- "vehicle.audi.tt": {
- "location": [-0.00037109359982423484, 1.3271345133958334e-10, 0.6926488876342773],
- "extent": [2.0906050205230713, 0.9970585703849792, 0.6926480531692505],
- "num_wheel": 4,
- },
- "vehicle.micro.microlino": {
- "location": [-2.0954755292068583e-11, -5.8593730500433594e-05, 0.6771557331085205],
- "extent": [1.1036475896835327, 0.7404598593711853, 0.6880123615264893],
- "num_wheel": 4,
- },
- "vehicle.bmw.grandtourer": {
- "location": [-1.885927941591703e-10, -0.0005273435963317752, 0.7651049494743347],
- "extent": [2.3055028915405273, 1.1208566427230835, 0.8336379528045654],
- "num_wheel": 4,
- },
- "vehicle.mercedes.sprinter": {
- "location": [-0.010537102818489075, 0.004140627570450306, 1.2882421016693115],
- "extent": [2.957595109939575, 0.9942164421081543, 1.2803276777267456],
- "num_wheel": 4,
- },
- "vehicle.dodge.charger_police": {
- "location": [0.023632802069187164, 0.0006054601399227977, 0.7926709055900574],
- "extent": [2.487122058868408, 1.0192005634307861, 0.7710590958595276],
- "num_wheel": 4,
- },
- "vehicle.jeep.wrangler_rubicon": {
- "location": [0.00021484403987415135, 0.00103515584487468, 0.9322753548622131],
- "extent": [1.9331103563308716, 0.9525982737541199, 0.9389679431915283],
- "num_wheel": 4,
- },
- "vehicle.mini.cooper_s": {
- "location": [-2.0954755292068583e-11, -5.8593730500433594e-05, 0.6930102109909058],
- "extent": [1.9029000997543335, 0.985137939453125, 0.7375151515007019],
- "num_wheel": 4,
- },
- "vehicle.nissan.micra": {
- "location": [3.492459099696532e-10, 0.0009765622089616954, 0.7812451124191284],
- "extent": [1.8166879415512085, 0.9225568771362305, 0.7506412863731384],
- "num_wheel": 4,
- },
- "vehicle.carlamotors.carlacola": {
- "location": [0.0, 0.0, 1.2337206602096558],
- "extent": [2.601919174194336, 1.3134948015213013, 1.2337223291397095],
- "num_wheel": 4,
- },
- "vehicle.mercedes.coupe_2020": {
- "location": [-0.00259765493683517, 9.289941038659322e-10, 0.7203612923622131],
- "extent": [2.3368194103240967, 1.0011461973190308, 0.7209736704826355],
- "num_wheel": 4,
- },
- "vehicle.dodge.charger_police_2020": {
- "location": [0.109501913189888, -3.916094470923781e-08, 0.8174071907997131],
- "extent": [2.6187572479248047, 1.0485419034957886, 0.819191575050354],
- "num_wheel": 4,
- },
- "vehicle.mini.cooper_s_2021": {
- "location": [-0.031249990686774254, 1.1175869119028903e-08, 0.8910351395606995],
- "extent": [2.2763495445251465, 1.0485360622406006, 0.8835831880569458],
- "num_wheel": 4,
- },
- "vehicle.bh.crossbike": {
- "location": [0.00029444904066622257, 195.28602600097656, 0.5411913990974426],
- "extent": [0.7436444163322449, 0.0, 0.5397894978523254],
- "num_wheel": 2,
- },
- "vehicle.nissan.patrol": {
- "location": [-0.05737302824854851, 0.00015627048560418189, 0.9351245164871216],
- "extent": [2.3022549152374268, 0.9657964706420898, 0.9274230599403381],
- "num_wheel": 4,
- },
- "vehicle.kawasaki.ninja": {
- "location": [-0.01960790529847145, 195.23898315429688, 0.5854394435882568],
- "extent": [1.0166761875152588, 0.0, 0.5727267861366272],
- "num_wheel": 2,
- },
- "vehicle.lincoln.mkz_2020": {
- "location": [-0.006289059761911631, 2.249143760124639e-09, 0.7437793016433716],
- "extent": [2.44619083404541, 1.115301489830017, 0.7400735020637512],
- "num_wheel": 4,
- },
- "vehicle.lincoln.mkz_2017": {
- "location": [0.004042967688292265, -1.4458780839277097e-09, 0.718859851360321],
- "extent": [2.4508416652679443, 1.0641621351242065, 0.7553732395172119],
- "num_wheel": 4,
- },
- "vehicle.ford.mustang": {
- "location": [0.03213866055011749, -1.149368333841494e-08, 0.6504687070846558],
- "extent": [2.358762502670288, 0.947413444519043, 0.650469958782196],
- "num_wheel": 4,
- },
- "vehicle.tesla.cybertruck": {
- "location": [0.0, 0.0, 1.049096703529358],
- "extent": [3.1367764472961426, 1.1947870254516602, 1.049095630645752],
- "num_wheel": 4,
- },
- "vehicle.ford.ambulance": {
- "location": [-0.2885546088218689, 0.001933696330524981, 1.2156102657318115],
- "extent": [3.18282151222229, 1.1755871772766113, 1.215687870979309],
- "num_wheel": 4,
- },
- "vehicle.mercedes.coupe": {
- "location": [-2.7939672797572257e-10, -0.0007812496623955667, 0.6667944192886353],
- "extent": [2.5133883953094482, 1.0757731199264526, 0.8253258466720581],
- "num_wheel": 4,
- },
- "vehicle.seat.leon": {
- "location": [-0.00014648489013779908, -0.0015624993247911334, 0.7396337985992432],
- "extent": [2.0964150428771973, 0.9080929160118103, 0.7369155883789062],
- "num_wheel": 4,
- },
- "vehicle.nissan.patrol_2021": {
- "location": [0.028378896415233612, -1.0149086016042475e-08, 1.0167187452316284],
- "extent": [2.782914400100708, 1.0749834775924683, 1.0225735902786255],
- "num_wheel": 4,
- },
- "vehicle.vespa.zx125": {
- "location": [0.010460472665727139, 195.32354736328125, 0.6211523413658142],
- "extent": [0.9023334980010986, 0.0, 0.6178141832351685],
- "num_wheel": 2,
- },
- "vehicle.yamaha.yzf": {
- "location": [-0.004266120493412018, 195.2182159423828, 0.64984130859375],
- "extent": [1.1047229766845703, 0.0, 0.6255727410316467],
- "num_wheel": 2,
- },
- "vehicle.chevrolet.impala": {
- "location": [5.859437442268245e-05, 0.0017968743341043591, 0.70149165391922],
- "extent": [2.6787397861480713, 1.0166014432907104, 0.7053293585777283],
- "num_wheel": 4,
- },
- "vehicle.volkswagen.t2": {
- "location": [0.0013281241990625858, -0.0005468753515742719, 1.0144726037979126],
- "extent": [2.2402184009552, 1.034657597541809, 1.0188959836959839],
- "num_wheel": 4,
- },
- "vehicle.citroen.c3": {
- "location": [8.451751520865969e-10, 0.002363280626013875, 0.76191645860672],
- "extent": [1.9938424825668335, 0.9254241585731506, 0.8085547685623169],
- "num_wheel": 4,
- },
- "vehicle.diamondback.century": {
- "location": [1.1252891454205383e-05, 195.3052520751953, 0.525390625],
- "extent": [0.8214218020439148, 0.0, 0.5119513869285583],
- "num_wheel": 2,
- },
- "vehicle.tesla.model3": {
- "location": [0.02921873889863491, -1.0449437759518787e-08, 0.7355297803878784],
- "extent": [2.3958897590637207, 1.081725001335144, 0.744159996509552],
- "num_wheel": 4,
- },
- "vehicle.carlamotors.firetruck": {
- "location": [-0.25343742966651917, 0.005312588065862656, 1.9133495092391968],
- "extent": [4.234020709991455, 1.4455441236495972, 1.9137061834335327],
- "num_wheel": 4,
- },
- "vehicle.audi.etron": {
- "location": [0.0, 0.0, 0.8205029368400574],
- "extent": [2.427854299545288, 1.0163782835006714, 0.8246796727180481],
- "num_wheel": 4,
- },
- "vehicle.dodge.charger_2020": {
- "location": [-0.006230466067790985, 2.228188966668654e-09, 0.76557856798172],
- "extent": [2.5030298233032227, 1.0485419034957886, 0.7673624753952026],
- "num_wheel": 4,
- },
- "vehicle.gazelle.omafiets": {
- "location": [-0.01564301736652851, 195.3675537109375, 0.5801635384559631],
- "extent": [0.9177202582359314, 0.0, 0.5628286004066467],
- "num_wheel": 2,
- },
- "vehicle.harley-davidson.low_rider": {
- "location": [-0.008875441737473011, 195.3657989501953, 0.6599145531654358],
- "extent": [1.1778701543807983, 0.0, 0.6382853388786316],
- "num_wheel": 2,
- },
- "vehicle.toyota.prius": {
- "location": [0.001992186764255166, -0.00039062555879354477, 0.724169909954071],
- "extent": [2.256761312484741, 1.0034072399139404, 0.7624167203903198],
- "num_wheel": 4,
- },
-}
-
-TOWN_DICTIONARY = {
- "2021_08_20_21_48_35": "Town06",
- "2021_08_18_19_48_05": "Town06",
- "2021_08_20_21_10_24": "Town06",
- "2021_08_21_09_28_12": "Town06",
- "2021_08_22_07_52_02": "Town05",
- "2021_08_22_09_08_29": "Town05",
- "2021_08_22_21_41_24": "Town05",
- "2021_08_23_12_58_19": "Town05",
- "2021_08_23_15_19_19": "Town04",
- "2021_08_23_16_06_26": "Town04",
- "2021_08_23_17_22_47": "Town04",
- "2021_08_23_21_07_10": "Town10HD",
- "2021_08_23_21_47_19": "Town10HD",
- "2021_08_24_07_45_41": "Town10HD",
- "2021_08_24_11_37_54": "Town07",
- "2021_08_24_20_09_18": "Town04",
- "2021_08_24_20_49_54": "Town04",
- "2021_08_24_21_29_28": "Town04",
- "2021_08_16_22_26_54": "Town06",
- "2021_08_18_09_02_56": "Town06",
- "2021_08_18_18_33_56": "Town06",
- "2021_08_18_21_38_28": "Town06",
- "2021_08_18_22_16_12": "Town06",
- "2021_08_18_23_23_19": "Town06",
- "2021_08_19_15_07_39": "Town06",
- "2021_08_20_16_20_46": "Town06",
- "2021_08_20_20_39_00": "Town06",
- "2021_08_20_21_00_19": "Town06",
- "2021_08_21_09_09_41": "Town06",
- "2021_08_21_15_41_04": "Town05",
- "2021_08_21_16_08_42": "Town05",
- "2021_08_21_17_00_32": "Town05",
- "2021_08_21_21_35_56": "Town05",
- "2021_08_21_22_21_37": "Town05",
- "2021_08_22_06_43_37": "Town05",
- "2021_08_22_07_24_12": "Town05",
- "2021_08_22_08_39_02": "Town05",
- "2021_08_22_09_43_53": "Town05",
- "2021_08_22_10_10_40": "Town05",
- "2021_08_22_10_46_58": "Town06",
- "2021_08_22_11_29_38": "Town06",
- "2021_08_22_22_30_58": "Town05",
- "2021_08_23_10_47_16": "Town04",
- "2021_08_23_11_06_41": "Town05",
- "2021_08_23_11_22_46": "Town04",
- "2021_08_23_12_13_48": "Town05",
- "2021_08_23_13_10_47": "Town05",
- "2021_08_23_16_42_39": "Town04",
- "2021_08_23_17_07_55": "Town04",
- "2021_08_23_19_27_57": "Town10HD",
- "2021_08_23_20_47_11": "Town10HD",
- "2021_08_23_22_31_01": "Town10HD",
- "2021_08_23_23_08_17": "Town10HD",
- "2021_08_24_09_25_42": "Town07",
- "2021_08_24_09_58_32": "Town07",
- "2021_08_24_12_19_30": "Town07",
- "2021_09_09_13_20_58": "Town03",
- "2021_09_09_19_27_35": "Town01",
- "2021_09_10_12_07_11": "Town04",
- "2021_09_09_23_21_21": "Town03",
- "2021_08_21_17_30_41": "Town05",
- "2021_08_22_13_37_16": "Town06",
- "2021_08_22_22_01_17": "Town05",
- "2021_08_23_10_51_24": "Town05",
- "2021_08_23_13_17_21": "Town05",
- "2021_08_23_19_42_07": "Town10HD",
- "2021_09_09_22_21_11": "Town02",
- "2021_09_11_00_33_16": "Town10HD",
- "2021_08_18_19_11_02": "Town06",
-}
-
-
-class bcolors:
- WARNING = "\033[93m"
- FAIL = "\033[91m"
- ENDC = "\033[0m"
-
-
-EXCLUDE_ROAD_MAP = [
- "2021_08_18_19_11_02",
- "2021_08_21_09_09_41",
- "2021_08_23_16_42_39",
- "2021_08_23_17_07_55",
- "2021_08_24_12_19_30",
- "2021_08_24_09_25_42",
- "2021_08_24_09_58_32",
- "2021_08_24_12_19_30",
- "2021_08_21_09_28_12",
- "2021_08_24_11_37_54",
- "2021_08_23_17_22_47",
- "2021_08_24_20_09_18",
-]
-
-OR_Z_VALUE_MAP = [
- "2021_08_22_09_43_53",
- "2021_08_22_10_10_40",
- "2021_08_23_12_13_48",
- "2021_08_23_13_10_47",
- "2021_08_24_09_25_42",
- "2021_08_24_09_58_32",
- "2021_08_24_12_19_30",
- "2021_09_09_23_21_21",
- "2021_09_10_12_07_11",
- "2021_08_22_09_08_29",
- "2021_08_23_12_58_19",
- "2021_08_23_17_22_47",
- "2021_08_24_11_37_54",
- "2021_08_24_20_09_18",
-]
diff --git a/OpenCOOD/logreplay/assets/utils.py b/OpenCOOD/logreplay/assets/utils.py
deleted file mode 100644
index 2d59d938..00000000
--- a/OpenCOOD/logreplay/assets/utils.py
+++ /dev/null
@@ -1,26 +0,0 @@
-from logreplay.assets.presave_lib import TOWN_DICTIONARY, BLUE_PRINT_LIB
-
-
-def find_town(scenario_name):
- """
- Given the scenario name, find the corresponding town name,
-
- Parameters
- ----------
- scenario_name : str
- The scenario name.
-
- Returns
- -------
- The corresponding town's name.
- """
- return TOWN_DICTIONARY[scenario_name]
-
-
-def find_blue_print(extent):
- for blueprint_name, blueprint_content in BLUE_PRINT_LIB.items():
- bp_extent = blueprint_content["extent"]
- if abs(extent[0] - bp_extent[0]) < 0.001 and abs(extent[1] - bp_extent[1]) < 0.001:
- return blueprint_name
-
- return None
diff --git a/OpenCOOD/logreplay/hypes_yaml/__init__.py b/OpenCOOD/logreplay/hypes_yaml/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/OpenCOOD/logreplay/hypes_yaml/replay.yaml b/OpenCOOD/logreplay/hypes_yaml/replay.yaml
deleted file mode 100644
index a38d05d8..00000000
--- a/OpenCOOD/logreplay/hypes_yaml/replay.yaml
+++ /dev/null
@@ -1,61 +0,0 @@
-root_dir: '/data/opv2v/train' # your original opv2v dataset
-output_dir: '/data/opv2v/train/additional' # the new folder to save the dumped new data
-map: # corresponding to bev map
- activate: true # whether activate HDMap module
- visualize: true # whether to visualize the BEV Map for each CAV
-
- save_yml: false # whether to save the map meta data in yaml files
- save_static: false # whether to dump road segmentation mask
- save_lane: true # whether to dump lane segmentation mask
- save_dynamic: false # whether to dump dynamic object BEV segmentaition mask
- save_bev_vis: true # whether to dump the BEV visualization image that has all elements
- save_bev_sem: true # whether to dump the BEV semantic mask that contains all elements
-
- radius: 50 # the BEV Map range, e.g., 50 means a square around the ego whose side length is 100
- raster_size: [256, 256] # the dumped BEV images resolution
- lane_sample_resolution: 0.1 # the lane keypoint sampling resolution
-
- static:
- draw_lane: true # whether to draw lane on the BEV visualization image
- draw_traffic_light: true # if set to true, the lanes that are influenced by traffic light will change color according to the light state, e.g., yellow, red, green
- exclude_road: false # set this to false by default
- exclude_intersection_lane: true # whether to draw the lane topology in the intersection
- z_filter_value: 6 # the road that is higher or lower relative to ego's position than the threshold won't be drawed
- other_objs: ['terrain', 'building', 'sidewalk'] # besides, lane and road, what other objects need to render on the BEV visualization image
- dynamic:
- exclude_self: true # if set to true, ego won't be drawed
- exclude_off_road: true # eclude the objects that are off road
- visibility: false # whether save visibility mask under ego coordinate. when set to true, semantic lidar has to be implemented
- visibility_corp: false # whether save visibility mask that can be seen for all cavs. when set to true, semantic lidar has to be implemented
-
-sensor: # new attached sensor
- sensor_list:
- - name: 'semantic_lidar'
- args: &base_semantic_lidar
- upper_fov: 10
- lower_fov: -30
- channels: 32
- rotation_frequency: 20
- points_per_second: 250000
- range: 50
- relative_pose: front
- thresh: 5 # mininum number of points hit to be regarded as visible
- - name: 'semantic_lidar'
- args:
- <<: *base_semantic_lidar
- relative_pose: left
- - name: 'semantic_lidar'
- args:
- <<: *base_semantic_lidar
- relative_pose: right
- - name: 'semantic_lidar'
- args:
- <<: *base_semantic_lidar
- relative_pose: back
- - name: 'bev_semantic_camera'
- args:
- visualize: true
- fov: 45
- image_size_x: 512
- image_size_y: 512
- height: 115.88
diff --git a/OpenCOOD/logreplay/map/__init__.py b/OpenCOOD/logreplay/map/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/OpenCOOD/logreplay/map/map_drawing.py b/OpenCOOD/logreplay/map/map_drawing.py
deleted file mode 100644
index acd04c36..00000000
--- a/OpenCOOD/logreplay/map/map_drawing.py
+++ /dev/null
@@ -1,190 +0,0 @@
-"""Rasterization drawing functions"""
-
-import numpy as np
-import cv2
-
-# sub-pixel drawing precision constants
-CV2_SUB_VALUES = {"shift": 9, "lineType": cv2.LINE_AA}
-CV2_SHIFT_VALUE = 2 ** CV2_SUB_VALUES["shift"]
-
-AGENT_COLOR = (255, 255, 255)
-ROAD_COLOR = (255, 255, 255)
-ROAD_COLOR_VIS = (17, 17, 31)
-Lane_COLOR = {"normal": (255, 217, 82), "red": (255, 0, 0), "yellow": (255, 255, 0), "green": (0, 255, 0)}
-BUILD_COLOR = (70, 70, 70)
-Terrain_COLOR = (145, 170, 100)
-SideWalk_COLOR = (244, 35, 232)
-# color map
-OBJ_COLOR_MAP = {"building": BUILD_COLOR, "terrain": Terrain_COLOR, "sidewalk": SideWalk_COLOR}
-
-
-def cv2_subpixel(coords: np.ndarray) -> np.ndarray:
- """
- Cast coordinates to numpy.int but keep fractional part by previously multiplying by 2**CV2_SHIFT
- cv2 calls will use shift to restore original values with higher precision
-
- Args:
- coords (np.ndarray): XY coords as float
-
- Returns:
- np.ndarray: XY coords as int for cv2 shift draw
- """
- coords = coords * CV2_SHIFT_VALUE
- coords = coords.astype(int)
- return coords
-
-
-def draw_agent(agent_list, image):
- """
- Draw agent mask on image.
-
- Parameters
- ----------
- agent_list : list
- The agent corner list.
-
- image : np.ndarray
- The image to be drawn.
-
- Returns
- -------
- Drawn image.
- """
- for agent_corner in agent_list:
- agent_corner = agent_corner.reshape(-1, 2)
- cv2.fillPoly(image, [agent_corner], AGENT_COLOR, **CV2_SUB_VALUES)
- return image
-
-
-def draw_road(lane_area_list, image, visualize=False):
- """
- Draw poly for road.
-
- Parameters
- ----------
- visualize : bool
- If set to true, the road segment will be black color.
-
- lane_area_list : list
- List of lane coordinates
-
- image : np.ndarray
- image to be drawn
-
- Returns
- -------
- drawed image.
- """
- color = ROAD_COLOR if not visualize else ROAD_COLOR_VIS
-
- for lane_area in lane_area_list:
- lane_area = lane_area.reshape(-1, 2)
- cv2.fillPoly(image, [lane_area], color, **CV2_SUB_VALUES)
- return image
-
-
-def road_exclude(static_road):
- """
- Exclude the road segment that is not connected to the ego vehicle
- position.
-
- Parameters
- ----------
- static_road : np.ndarray
- The static bev map with road segment.
-
- Returns
- -------
- The road without unrelated road.
- """
- binary_bev = cv2.cvtColor(static_road, cv2.COLOR_BGR2GRAY)
- _, label, _, _ = cv2.connectedComponentsWithStats(binary_bev)
-
- ego_label = label[static_road.shape[0] // 2, static_road.shape[1] // 2]
- static_road[label != ego_label] = 0
-
- return static_road
-
-
-def draw_lane(lane_area_list, lane_type_list, image, intersection_list=None, vis=True):
- """
- Draw lanes on image (polylines).
-
- Parameters
- ----------
- intersection_list : list
- lane_area_list : list
- List of lane coordinates
-
- lane_type_list : list
- List of lane types, normal, red, green or yellow.
-
- image : np.ndarray
- image to be drawn
-
- vis : bool
- Whether to visualize
-
- Returns
- -------
- drawed image.
- """
- if intersection_list is None:
- intersection_list = [False] * len(lane_area_list)
-
- for lane_area, lane_type, inter_flag in zip(lane_area_list, lane_type_list, intersection_list):
- if inter_flag:
- continue
- cv2.polylines(image, lane_area, False, Lane_COLOR[lane_type] if vis else (255, 255, 255), **CV2_SUB_VALUES)
-
- return image
-
-
-def draw_crosswalks(lane_area_list, image):
- """
- Draw lanes on image (polylines).
-
- Parameters
- ----------
- lane_area_list : list
- List of cross coordinates
-
- image : np.ndarray
- image to be drawn
-
- vis : bool
- Whether to visualize
-
- Returns
- -------
- drawed image.
- """
- for lane_area in lane_area_list:
- up_line = lane_area[0]
- bottom_line = lane_area[1]
-
- cv2.line(image, (up_line[0, 0], up_line[0, 1]), (up_line[-1, 0], up_line[-1, 1]), (255, 255, 255), 2, **CV2_SUB_VALUES)
- cv2.line(image, (bottom_line[0, 0], bottom_line[0, 1]), (bottom_line[-1, 0], bottom_line[-1, 1]), (255, 255, 255), 2, **CV2_SUB_VALUES)
- cv2.line(image, (up_line[0, 0], up_line[0, 1]), (bottom_line[-1, 0], bottom_line[-1, 1]), (255, 255, 255), 2, **CV2_SUB_VALUES)
- cv2.line(image, (up_line[-1, 0], up_line[-1, 1]), (bottom_line[0, 0], bottom_line[0, 1]), (255, 255, 255), 2, **CV2_SUB_VALUES)
- return image
-
-
-def draw_city_objects(city_obj_info, image):
- """
- Draw static objects other than lane, road, crosswalks on image.
-
- Parameters
- ----------
- city_obj_info : dict
- image : np.ndarray
-
- Returns
- -------
- Drew image.
- """
- for obj_category, obj_content in city_obj_info.items():
- for _, obj in obj_content.items():
- obj_corner = obj["corner_area"].reshape(-1, 2)
- cv2.fillPoly(image, [obj_corner], OBJ_COLOR_MAP[obj_category], **CV2_SUB_VALUES)
- return image
diff --git a/OpenCOOD/logreplay/map/map_manager.py b/OpenCOOD/logreplay/map/map_manager.py
deleted file mode 100644
index 46a9ef0a..00000000
--- a/OpenCOOD/logreplay/map/map_manager.py
+++ /dev/null
@@ -1,841 +0,0 @@
-"""HDMap manager"""
-
-import math
-import os.path
-import uuid
-
-import cv2
-import carla
-import numpy as np
-from matplotlib.path import Path
-from shapely.geometry import Polygon
-
-from logreplay.assets.presave_lib import EXCLUDE_ROAD_MAP, OR_Z_VALUE_MAP
-from logreplay.map.map_utils import (
- world_to_sensor,
- lateral_shift,
- list_loc2array,
- list_wpt2array,
- convert_tl_status,
- exclude_off_road_agents,
- retrieve_city_object_info,
- obj_in_range,
-)
-from logreplay.map.map_drawing import cv2_subpixel, draw_agent, draw_road, draw_lane, road_exclude, draw_crosswalks, draw_city_objects
-from opencood.hypes_yaml.yaml_utils import save_yaml_wo_overwriting
-
-CV2_SUB_VALUES = {"shift": 9, "lineType": cv2.LINE_AA}
-
-
-class MapManager(object):
- """
- This class is used to manage HD Map. We emulate the style of Lyft dataset.
-
- Parameters
- ----------
- world : Carla.World
- CARLA world.
-
- config : dict
- All the map manager parameters.
-
- output_root : str
- The data dump root folder.
-
- scene_name : str
- The name of the scene.
-
- Attributes
- ----------
- world : carla.world
- Carla simulation world.
-
- meter_per_pixel : float
- m/pixel
-
- raster_size : float
- The rasterization map size in pixels.
-
- radius_meter : float
- The valid radius(m) in the center of the rasterization map.
-
- topology : list
- Map topology in list.
-
- lane_info : dict
- A dictionary that contains all lane information.
-
- crosswalk_info : dict
- A dictionary that contains all crosswalk information.
- todo: will be implemented in the next version.
-
- traffic_light_info : dict
- A dictionary that contains all traffic light information.
-
- bound_info : dict
- A dictionary that saves boundary information of lanes and crosswalks.
- It is used to efficiently filter out invalid lanes/crosswarlks.
-
- lane_sample_resolution : int
- The sampling resolution for drawing lanes.
-
- draw_lane : bool
- Whether to draw lane on the static bev map
-
- static_bev : np.array
- The static bev map containing lanes and drivable road information.
-
- dynamic_bev : np.array
- The dynamic bev map containing vehicle's information.
-
- vis_bev : np.array
- The comprehensive bev map for visualization.
-
- """
-
- def __init__(self, world, config, output_root, scene_name):
- self.world = world
- self.carla_map = world.get_map()
- self.center = None
- self.actor_id = None
- self.out_root = output_root
- self.scene_name = scene_name
-
- # whether to activate this module
- self.activate = config["activate"]
- if not self.activate:
- return
-
- # save flags
- self.save_yml = config["save_yml"]
- self.save_static = config["save_static"]
- self.save_dynamic = config["save_dynamic"]
- self.save_lane = config["save_lane"]
- self.save_bev_vis = config["save_bev_vis"]
-
- # whether to visualize the bev map while running simulation
- self.visualize = config["visualize"]
- # whether exclude the road that is unrelated to the ego vehicle
- self.exclude_road = config["static"]["exclude_road"]
- if self.scene_name in EXCLUDE_ROAD_MAP:
- self.exclude_road = True
-
- self.radius_meter = config["radius"]
- self.z_filter_value = config["static"]["z_filter_value"]
- self.exclude_intersection_lane = config["static"]["exclude_intersection_lane"]
- self.other_render_objs = config["static"]["other_objs"]
-
- assert config["raster_size"][0] == config["raster_size"][1]
- self.raster_size = np.array([config["raster_size"][0], config["raster_size"][1]])
- self.pixels_per_meter = self.raster_size[0] / (self.radius_meter * 2)
- self.meter_per_pixel = 1 / self.pixels_per_meter
- self.lane_sample_resolution = config["lane_sample_resolution"]
-
- # list of all start waypoints in HD Map
- topology = [x[0] for x in self.carla_map.get_topology()]
- # sort by altitude
- self.topology = sorted(topology, key=lambda w: w.transform.location.z)
-
- # basic elements in HDMap: lane, crosswalk and traffic light
- self.lane_info = {}
- self.crosswalk_info = {}
- self.traffic_light_info = {}
- # this is mainly used for efficient filtering
- self.bound_info = {"lanes": {}, "crosswalks": {}}
- self.traffic_stop_pos = []
-
- self.retrieve_light_stop_pos()
- # generate information for traffic light
- self.generate_tl_info(self.world)
- # generate lane, crosswalk and boundary information
- self.generate_lane_cross_info()
-
- # generate labels
- self.other_objs_info = retrieve_city_object_info(self.world, self.other_render_objs)
-
- # static related info
- self.draw_lane = config["static"]["draw_lane"]
- self.draw_traffic_light = config["static"]["draw_traffic_light"]
- # dynamic related info
- self.exclude_self = config["dynamic"]["exclude_self"]
- self.exclude_off_road = config["dynamic"]["exclude_off_road"]
-
- # whether to check visibility in camera for ego only
- self.visibility = config["dynamic"]["visibility"]
- # visible agent id for ego
- self.vis_ids = []
-
- # whether to check visibility in camera for multi-agent perspective
- self.visibility_corp = config["dynamic"]["visibility_corp"]
- # visible agent id for all vehicles
- self.vis_corp_ids = []
-
- # bev maps
- self.dynamic_bev = 255 * np.zeros(shape=(self.raster_size[1], self.raster_size[0], 3), dtype=np.uint8)
- self.vis_mask = 255 * np.zeros(shape=(self.raster_size[1], self.raster_size[0], 3), dtype=np.uint8)
- self.vis_corp_mask = 255 * np.zeros(shape=(self.raster_size[1], self.raster_size[0], 3), dtype=np.uint8)
- self.static_bev = 255 * np.zeros(shape=(self.raster_size[1], self.raster_size[0], 3), dtype=np.uint8)
- self.lane_bev = 255 * np.zeros(shape=(self.raster_size[1], self.raster_size[0], 3), dtype=np.uint8)
- self.vis_bev = 255 * np.ones(shape=(self.raster_size[1], self.raster_size[0], 3), dtype=np.uint8)
-
- def run_step(self, cav_id, cav_content, veh_dict):
- """
- Rasterization + Visualize the bev map if needed.
-
- Parameters
- ----------
- cav_id : str
- The cav's original id in the dataset.
- cav_content : dict
- The cav's information.
- veh_dict : dict
- All vehicles' information. Used for visibility check for coop.
- """
- if not self.activate:
- return
-
- self.actor_id = cav_content["actor_id"]
- self.center = cav_content["cur_pose"]
- self.agent_id = cav_id
- self.current_timstamp = cav_content["cur_count"]
-
- # clean buffer every round starts
- self.vis_ids = []
- self.vis_corp_ids = []
-
- if self.visibility:
- self.vis_ids = self.check_visibility_single(cav_content, self.vis_ids)
- if self.visibility_corp:
- self.vis_corp_ids = self.check_visibility_corp(veh_dict, self.vis_corp_ids)
-
- self.rasterize_static()
- self.rasterize_dynamic()
-
- if self.exclude_off_road:
- self.dynamic_bev = exclude_off_road_agents(self.static_bev, self.dynamic_bev)
-
- if self.visualize:
- cv2.imshow("the bev map of agent %s" % self.agent_id, self.vis_bev)
- cv2.waitKey(1)
- self.data_dump()
-
- def check_visibility_corp(self, veh_dict, vis_corp_mask_list):
- for _, veh_contnet in veh_dict.items():
- if "cav" in veh_contnet:
- vis_corp_mask_list += self.check_visibility_single(veh_contnet, vis_corp_mask_list)
- vis_corp_mask_list = list(set(vis_corp_mask_list))
- return vis_corp_mask_list
-
- # check visibility agent list for ego vehicle
- @staticmethod
- def check_visibility_single(cav_content, vis_mask_list):
- """
- Retrieve visible objects in the ego camera.
-
- Parameters
- ----------
- vis_mask_list : list
- cav_content : dict
- """
-
- sensor_manager = cav_content["sensor_manager"]
- meta_data = sensor_manager.sensor_meta
- for key, item in meta_data.items():
- if "semantic_lidar" in key:
- vis_mask_list += item
- vis_mask_list = list(set(vis_mask_list))
- return vis_mask_list
-
- @staticmethod
- def get_bounds(left_lane, right_lane):
- """
- Get boundary information of a lane.
-
- Parameters
- ----------
- left_lane : np.array
- shape: (n, 3)
- right_lane : np.array
- shape: (n,3)
- Returns
- -------
- bound : np.array
- """
- x_min = min(np.min(left_lane[:, 0]), np.min(right_lane[:, 0]))
- y_min = min(np.min(left_lane[:, 1]), np.min(right_lane[:, 1]))
- z_min = min(np.min(left_lane[:, 2]), np.min(right_lane[:, 2]))
- x_max = max(np.max(left_lane[:, 0]), np.max(right_lane[:, 0]))
- y_max = max(np.max(left_lane[:, 1]), np.max(right_lane[:, 1]))
- z_max = max(np.max(left_lane[:, 2]), np.max(right_lane[:, 2]))
-
- bounds = np.asarray([[[x_min, y_min], [x_max, y_max], [z_min, z_max]]])
-
- return bounds
-
- def agents_in_range(self, radius, agents_dict):
- """
- Filter out all agents out of the radius.
-
- Parameters
- ----------
- radius : float
- Radius in meters
-
- agents_dict : dict
- Dictionary containing all dynamic agents.
-
- Returns
- -------
- The dictionary that only contains the agent in range.
- """
- final_agents = {}
-
- # convert center to list format
- center = [self.center.location.x, self.center.location.y]
-
- for agent_id, agent in agents_dict.items():
- location = agent["location"]
- distance = math.sqrt((location[0] - center[0]) ** 2 + (location[1] - center[1]) ** 2)
- if distance < radius:
- final_agents.update({agent_id: agent})
-
- return final_agents
-
- def indices_in_bounds(self, bounds: np.ndarray, half_extent: float) -> np.ndarray:
- """
- Get indices of elements for which the bounding box described by bounds
- intersects the one defined around center (square with side 2*half_side)
-
- Parameters
- ----------
- bounds :np.ndarray
- array of shape Nx2x2 [[x_min,y_min],[x_max, y_max]]
-
- half_extent : float
- half the side of the bounding box centered around center
-
- Returns
- -------
- np.ndarray: indices of elements inside radius from center
- """
- x_center, y_center, z_center = self.center.location.x, self.center.location.y, self.center.location.z
-
- x_min_in = x_center > bounds[:, 0, 0] - half_extent
- y_min_in = y_center > bounds[:, 0, 1] - half_extent
- x_max_in = x_center < bounds[:, 1, 0] + half_extent
- y_max_in = y_center < bounds[:, 1, 1] + half_extent
-
- z_min_in = abs(z_center - bounds[:, 2, 0]) < self.z_filter_value
- z_max_in = abs(z_center - bounds[:, 2, 1]) < self.z_filter_value
-
- # this several scenes has some speciality and needs to be hardcoded
- if self.scene_name in OR_Z_VALUE_MAP:
- z_flag = z_min_in | z_max_in
- else:
- z_flag = z_min_in & z_max_in
-
- return np.nonzero(x_min_in & y_min_in & x_max_in & y_max_in & z_flag)[0]
-
- def retrieve_light_stop_pos(self):
- # retrieve all stop and traffic light position
- all_actors = self.world.get_actors()
- for actor in all_actors:
- if "traffic_light" in actor.type_id or "stop" in actor.type_id:
- location = actor.get_transform().location
- self.traffic_stop_pos.append((location.x, location.y))
-
- def associate_lane_tl(self, mid_lane):
- """
- Given the waypoints for a certain lane, find the traffic light that
- influence it.
-
- Parameters
- ----------
- mid_lane : np.ndarray
- The middle line of the lane.
- Returns
- -------
- associate_tl_id : str
- The associated traffic light id.
- """
- associate_tl_id = ""
-
- for tl_id, tl_content in self.traffic_light_info.items():
- trigger_poly = tl_content["corners"]
- # use Path to do fast computation
- trigger_path = Path(trigger_poly.boundary)
- # check if any point in the middle line inside the trigger area
- check_array = trigger_path.contains_points(mid_lane[:, :2])
-
- if check_array.any():
- associate_tl_id = tl_id
- return associate_tl_id
-
- def data_dump(self):
- """
- Dump the data to the corresponding folder.
- """
- save_name = os.path.join(self.out_root, self.agent_id)
-
- if not os.path.exists(save_name):
- os.makedirs(save_name)
-
- map_info = {
- "bev_map": {
- "raster_size_pixel": self.raster_size,
- "radius_meter": self.radius_meter,
- "pixel_per_meter": self.pixels_per_meter,
- "topology": self.valid_lane_info,
- }
- }
- if self.save_yml:
- # save metadata
- save_yaml_name = os.path.join(save_name, self.current_timstamp + "_additional.yaml")
- save_yaml_wo_overwriting(map_info, save_yaml_name)
-
- if self.save_static:
- # save rgb image
- save_static_name = os.path.join(save_name, self.current_timstamp + "_bev_static.png")
- cv2.imwrite(save_static_name, self.static_bev)
-
- if self.save_lane:
- # save lane topology seg
- save_lane_name = os.path.join(save_name, self.current_timstamp + "_bev_lane.png")
- cv2.imwrite(save_lane_name, self.lane_bev)
-
- if self.save_dynamic:
- # save dynamic bev
- save_dynamic_name = os.path.join(save_name, self.current_timstamp + "_bev_dynamic.png")
- cv2.imwrite(save_dynamic_name, self.dynamic_bev)
-
- if self.visibility:
- # save visibility map for ego agent
- save_visibility_name = os.path.join(save_name, self.current_timstamp + "_bev_visibility.png")
- cv2.imwrite(save_visibility_name, self.vis_mask)
-
- if self.visibility_corp:
- # save visibility map for ego agent
- save_visibility_name = os.path.join(save_name, self.current_timstamp + "_bev_visibility_corp.png")
- cv2.imwrite(save_visibility_name, self.vis_corp_mask)
-
- if self.save_bev_vis:
- # save visualize bev
- save_vis_name = os.path.join(save_name, self.current_timstamp + "_bev_vis.png")
- cv2.imwrite(save_vis_name, self.vis_bev)
-
- def generate_lane_cross_info(self):
- """
- From the topology generate all lane and crosswalk
- information in a dictionary under world's coordinate frame.
- """
- # list of str
- lanes_id = []
- crosswalks_ids = []
-
- # boundary of each lane for later filtering
- lanes_bounds = np.empty((0, 3, 2), dtype=float)
- crosswalks_bounds = np.empty((0, 3, 2), dtype=float)
-
- # for crosswalk information
- crosswalks_list = self.split_cross_walks()
- for i, crosswalk in enumerate(crosswalks_list):
- crosswalk_id = uuid.uuid4().hex[:6].upper()
- crosswalks_ids.append(crosswalk_id)
-
- cross_marking = np.array(crosswalk)
- bound = self.get_bounds(cross_marking, cross_marking)
- crosswalks_bounds = np.append(crosswalks_bounds, bound, axis=0)
-
- self.crosswalk_info.update({crosswalk_id: {"xyz": cross_marking}})
-
- self.bound_info["crosswalks"]["ids"] = crosswalks_ids
- self.bound_info["crosswalks"]["bounds"] = crosswalks_bounds
-
- # loop all waypoints to get lane information
- for i, waypoint in enumerate(self.topology):
- # unique id for each lane
- lane_id = uuid.uuid4().hex[:6].upper()
- lanes_id.append(lane_id)
-
- intersection_flag = True if waypoint.is_intersection and self.exclude_intersection_lane else False
- # we need to exclude the intersections without traffic light/stops
- if intersection_flag:
- intersection_flag = False
- for t_pos in self.traffic_stop_pos:
- distance = np.sqrt((t_pos[0] - waypoint.transform.location.x) ** 2 + (t_pos[1] - waypoint.transform.location.y) ** 2)
- if distance <= 30:
- intersection_flag = True
- break
-
- waypoints = [waypoint]
- nxt = waypoint.next(self.lane_sample_resolution)[0]
- # looping until next lane
- while nxt.road_id == waypoint.road_id and nxt.lane_id == waypoint.lane_id:
- waypoints.append(nxt)
- nxt = nxt.next(self.lane_sample_resolution)[0]
-
- # waypoint is the centerline, we need to calculate left lane mark
- left_marking = [lateral_shift(w.transform, -w.lane_width * 0.5) for w in waypoints]
- right_marking = [lateral_shift(w.transform, w.lane_width * 0.5) for w in waypoints]
- # convert the list of carla.Location to np.array
- left_marking = list_loc2array(left_marking)
- right_marking = list_loc2array(right_marking)
- mid_lane = list_wpt2array(waypoints)
-
- # get boundary information
- bound = self.get_bounds(left_marking, right_marking)
- lanes_bounds = np.append(lanes_bounds, bound, axis=0)
-
- # associate with traffic light
- tl_id = self.associate_lane_tl(mid_lane)
-
- self.lane_info.update(
- {
- lane_id: {
- "xyz_left": left_marking,
- "xyz_right": right_marking,
- "xyz_mid": mid_lane,
- "tl_id": tl_id,
- "intersection_flag": intersection_flag,
- }
- }
- )
- # boundary information
- self.bound_info["lanes"]["ids"] = lanes_id
- self.bound_info["lanes"]["bounds"] = lanes_bounds
-
- def split_cross_walks(self):
- """
- Find each crosswalk with their key points.
- """
- all_cross_walks = self.carla_map.get_crosswalks()
- cross_walks_list = []
-
- tmp_list = []
- for key_points in all_cross_walks:
- if (key_points.x, key_points.y, key_points.z) in tmp_list:
- cross_walks_list.append(tmp_list)
- tmp_list = []
- else:
- tmp_list.append((key_points.x, key_points.y, key_points.z))
-
- return cross_walks_list
-
- def generate_tl_info(self, world):
- """
- Generate traffic light information under world's coordinate frame.
-
- Parameters
- ----------
- world : carla.world
- Carla simulator env.
-
- Returns
- -------
- A dictionary containing traffic lights information.
- """
- tl_list = world.get_actors().filter("traffic.traffic_light*")
-
- for tl_actor in tl_list:
- tl_id = uuid.uuid4().hex[:4].upper()
-
- base_transform = tl_actor.get_transform()
- base_rot = base_transform.rotation.yaw
- # this is where the vehicle stops
- area_loc = base_transform.transform(tl_actor.trigger_volume.location)
- area_transform = carla.Transform(area_loc, carla.Rotation(yaw=base_rot))
-
- # bbx extent
- ext = tl_actor.trigger_volume.extent
- # the y is to narrow
- ext.y += 0.5
- ext_corners = np.array([[-ext.x, -ext.y], [ext.x, -ext.y], [ext.x, ext.y], [-ext.x, ext.y]])
- for i in range(ext_corners.shape[0]):
- corrected_loc = area_transform.transform(carla.Location(ext_corners[i][0], ext_corners[i][1]))
- ext_corners[i, 0] = corrected_loc.x
- ext_corners[i, 1] = corrected_loc.y
-
- # use shapely lib to convert to polygon
- corner_poly = Polygon(ext_corners)
- self.traffic_light_info.update(
- {tl_id: {"actor": tl_actor, "corners": corner_poly, "base_rot": base_rot, "base_transform": base_transform}}
- )
-
- def generate_lane_area(self, xyz_left, xyz_right):
- """
- Generate the lane area poly under rasterization map's center
- coordinate frame.
-
- Parameters
- ----------
- xyz_left : np.ndarray
- Left lanemarking of a lane, shape: (n, 3).
- xyz_right : np.ndarray
- Right lanemarking of a lane, shape: (n, 3).
-
- Returns
- -------
- lane_area : np.ndarray
- Combine left and right lane together to form a polygon.
- """
- lane_area = np.zeros((2, xyz_left.shape[0], 2))
- # convert coordinates to center's coordinate frame
- xyz_left = xyz_left.T
- xyz_left = np.r_[xyz_left, [np.ones(xyz_left.shape[1])]]
- xyz_right = xyz_right.T
- xyz_right = np.r_[xyz_right, [np.ones(xyz_right.shape[1])]]
-
- # ego's coordinate frame
- xyz_left = world_to_sensor(xyz_left, self.center).T
- xyz_right = world_to_sensor(xyz_right, self.center).T
-
- # to image coordinate frame
- lane_area[0] = xyz_left[:, :2]
- lane_area[1] = xyz_right[::-1, :2]
- # switch x and y
- lane_area = lane_area[..., ::-1]
- # y revert
- lane_area[:, :, 1] = -lane_area[:, :, 1]
-
- lane_area[:, :, 0] = lane_area[:, :, 0] * self.pixels_per_meter + self.raster_size[0] // 2
- lane_area[:, :, 1] = lane_area[:, :, 1] * self.pixels_per_meter + self.raster_size[1] // 2
-
- # to make more precise polygon
- lane_area = cv2_subpixel(lane_area)
-
- return lane_area
-
- def generate_cross_area(self, xyz):
- """
- Generate the cross lane under rasterization map's center
- coordinate frame.
-
- Parameters
- ----------
- xyz : np.ndarray
- Crosswalk lane marking, shape: (n, 3).
- Returns
- -------
- lane_area : np.ndarray
- Combine up and down line together.
- """
- if xyz.shape[0] == 2:
- print("dman")
- lane_area = np.zeros((2, xyz.shape[0] // 2, 2))
- # convert coordinates to center's coordinate frame
- xyz = xyz.T
- xyz = np.r_[xyz, [np.ones(xyz.shape[1])]]
-
- # ego's coordinate frame
- xyz = world_to_sensor(xyz, self.center).T
-
- # to image coordinate frame
- lane_area[0] = xyz[: xyz.shape[0] // 2, :2]
- lane_area[1] = xyz[xyz.shape[0] // 2 :, :2]
- # switch x and y
- lane_area = lane_area[..., ::-1]
- # y revert
- lane_area[:, :, 1] = -lane_area[:, :, 1]
-
- lane_area[:, :, 0] = lane_area[:, :, 0] * self.pixels_per_meter + self.raster_size[0] // 2
- lane_area[:, :, 1] = lane_area[:, :, 1] * self.pixels_per_meter + self.raster_size[1] // 2
- # to make more precise polygon
- lane_area = cv2_subpixel(lane_area)
-
- return lane_area
-
- def generate_agent_area(self, corners):
- """
- Convert the agent's bbx corners from world coordinates to
- rasterization coordinates.
-
- Parameters
- ----------
- corners : list
- The four corners of the agent's bbx under world coordinate.
-
- Returns
- -------
- agent four corners in image.
- """
- # (4, 3) numpy array
- corners = np.array(corners)
- # for homogeneous transformation
- corners = corners.T
- corners = np.r_[corners, [np.ones(corners.shape[1])]]
- # convert to ego's coordinate frame
- corners = world_to_sensor(corners, self.center).T
- corners = corners[:, :2]
-
- # switch x and y
- corners = corners[..., ::-1]
- # y revert
- corners[:, 1] = -corners[:, 1]
-
- corners[:, 0] = corners[:, 0] * self.pixels_per_meter + self.raster_size[0] // 2
- corners[:, 1] = corners[:, 1] * self.pixels_per_meter + self.raster_size[1] // 2
-
- # to make more precise polygon
- corner_area = cv2_subpixel(corners[:, :2])
-
- return corner_area
-
- def load_agents_world(self):
- """
- Load all the dynamic agents info from server directly
- into a dictionary.
-
- Returns
- -------
- The dictionary contains all agents info in the carla world.
- """
-
- agent_list = self.world.get_actors().filter("vehicle.*")
- dynamic_agent_info = {}
-
- for agent in agent_list:
- agent_id = agent.id
-
- agent_transform = agent.get_transform()
- agent_loc = [
- agent_transform.location.x,
- agent_transform.location.y,
- agent_transform.location.z,
- ]
-
- agent_yaw = agent_transform.rotation.yaw
-
- # calculate 4 corners
- bb = agent.bounding_box.extent
- corners = [
- carla.Location(x=-bb.x, y=-bb.y),
- carla.Location(x=-bb.x, y=bb.y),
- carla.Location(x=bb.x, y=bb.y),
- carla.Location(x=bb.x, y=-bb.y),
- ]
- # corners are originally in ego coordinate frame, convert to
- # world coordinate
- agent_transform.transform(corners)
- corners_reformat = [[x.x, x.y, x.z] for x in corners]
-
- dynamic_agent_info[agent_id] = {"location": agent_loc, "yaw": agent_yaw, "corners": corners_reformat}
- return dynamic_agent_info
-
- def rasterize_dynamic(self):
- """
- Rasterize the dynamic agents.
-
- Returns
- -------
- Rasterization image.
- """
- self.dynamic_bev = 255 * np.zeros(shape=(self.raster_size[1], self.raster_size[0], 3), dtype=np.uint8)
- self.vis_mask = 255 * np.zeros(shape=(self.raster_size[1], self.raster_size[0], 3), dtype=np.uint8)
- self.vis_corp_mask = 255 * np.zeros(shape=(self.raster_size[1], self.raster_size[0], 3), dtype=np.uint8)
- # filter using half a radius from the center
- raster_radius = float(np.linalg.norm(self.raster_size * np.array([self.meter_per_pixel, self.meter_per_pixel]))) / 2
- # retrieve all agents
- dynamic_agents = self.load_agents_world()
- # filter out agents out of range
- final_agents = self.agents_in_range(raster_radius, dynamic_agents)
-
- corner_list = []
- vis_corner_list = []
- vis_corp_corner_list = []
- for agent_id, agent in final_agents.items():
- # in case we don't want to draw the cav itself
- if agent_id == self.actor_id and self.exclude_self:
- continue
- agent_corner = self.generate_agent_area(agent["corners"])
- corner_list.append(agent_corner)
-
- # get visibility for single vehicle
- if agent_id in self.vis_ids:
- vis_corner_list.append(agent_corner)
- # get visibility for multi-agents
- if agent_id in self.vis_corp_ids:
- vis_corp_corner_list.append(agent_corner)
-
- self.dynamic_bev = draw_agent(corner_list, self.dynamic_bev)
- self.vis_bev = draw_agent(corner_list, self.vis_bev)
- self.vis_mask = draw_agent(vis_corner_list, self.vis_mask)
- self.vis_corp_mask = draw_agent(vis_corp_corner_list, self.vis_corp_mask)
-
- def rasterize_static(self):
- """
- Generate the static bev map.
- """
- self.static_bev = 255 * np.zeros(shape=(self.raster_size[1], self.raster_size[0], 3), dtype=np.uint8)
- self.lane_bev = 255 * np.zeros(shape=(self.raster_size[1], self.raster_size[0], 3), dtype=np.uint8)
- self.vis_bev = 255 * np.ones(shape=(self.raster_size[1], self.raster_size[0], 3), dtype=np.uint8)
-
- # filter using half a radius from the center
- raster_radius = float(np.linalg.norm(self.raster_size * np.array([self.meter_per_pixel, self.meter_per_pixel]))) / 2
- lane_indices = self.indices_in_bounds(self.bound_info["lanes"]["bounds"], raster_radius)
- cross_indices = self.indices_in_bounds(self.bound_info["crosswalks"]["bounds"], raster_radius)
-
- # --------- Retrieve road topology related first ---------- #
- lanes_area_list = []
- cross_area_list = []
- lane_type_list = []
- intersection_list = []
-
- for idx, cross_idx in enumerate(cross_indices):
- cross_idx = self.bound_info["crosswalks"]["ids"][cross_idx]
- cross_info = self.crosswalk_info[cross_idx]
-
- cross_area = self.generate_cross_area(cross_info["xyz"])
- cross_area_list.append(cross_area)
-
- for idx, lane_idx in enumerate(lane_indices):
- lane_idx = self.bound_info["lanes"]["ids"][lane_idx]
- lane_info = self.lane_info[lane_idx]
-
- # save the topology info for this cav
- self.valid_lane_info = lane_info
-
- xyz_left, xyz_right = lane_info["xyz_left"], lane_info["xyz_right"]
-
- # generate lane area
- lane_area = self.generate_lane_area(xyz_left, xyz_right)
- lanes_area_list.append(lane_area)
-
- # intersection flag
- intersection_list.append(lane_info["intersection_flag"])
-
- # check the associated traffic light
- associated_tl_id = lane_info["tl_id"]
- if associated_tl_id and self.draw_traffic_light:
- tl_actor = self.traffic_light_info[associated_tl_id]["actor"]
- status = convert_tl_status(tl_actor.get_state())
- lane_type_list.append(status)
- else:
- lane_type_list.append("normal")
-
- self.static_bev = draw_road(lanes_area_list, self.static_bev)
- if self.exclude_road:
- self.static_bev = road_exclude(self.static_bev)
-
- if self.draw_lane:
- self.lane_bev = draw_lane(lanes_area_list, lane_type_list, self.lane_bev, intersection_list, vis=False)
- self.lane_bev = draw_crosswalks(cross_area_list, self.lane_bev)
- self.lane_bev[self.static_bev == 0] = 0
-
- # --------- Retrieve city objects such as buildings ---------- #
- # first filter out objects out of range
- final_city_objs = obj_in_range(self.center, raster_radius, self.other_objs_info)
- for _, obj_content in final_city_objs.items():
- for obj_id, obj in obj_content.items():
- corner_area = self.generate_agent_area(obj["corners"])
- obj["corner_area"] = corner_area
-
- # we try to draw everything for visualization, but only dumping the
- # elements we need.
- self.vis_bev = draw_city_objects(final_city_objs, self.vis_bev)
- self.vis_bev = draw_road(lanes_area_list, self.vis_bev, visualize=True)
- self.vis_bev = draw_lane(lanes_area_list, lane_type_list, self.vis_bev, intersection_list)
- self.vis_bev = draw_crosswalks(cross_area_list, self.vis_bev)
- self.vis_bev = cv2.cvtColor(self.vis_bev, cv2.COLOR_RGB2BGR)
-
- def destroy(self):
- cv2.destroyAllWindows()
diff --git a/OpenCOOD/logreplay/map/map_utils.py b/OpenCOOD/logreplay/map/map_utils.py
deleted file mode 100644
index dcbcabef..00000000
--- a/OpenCOOD/logreplay/map/map_utils.py
+++ /dev/null
@@ -1,241 +0,0 @@
-"""HDMap utilities"""
-
-import carla
-import numpy as np
-import uuid
-import math
-
-LABEL_TO_CARLA = {"building": carla.CityObjectLabel.Buildings, "terrain": carla.CityObjectLabel.Terrain, "sidewalk": carla.CityObjectLabel.Sidewalks}
-
-
-def lateral_shift(transform, shift):
- transform.rotation.yaw += 90
- return transform.location + shift * transform.get_forward_vector()
-
-
-def list_loc2array(list_location):
- """
- Convert list of carla location to np.array
- Parameters
- ----------
- list_location : list
- List of carla locations.
-
- Returns
- -------
- loc_array : np.array
- Numpy array of shape (N, 3)
- """
- loc_array = np.zeros((len(list_location), 3))
- for i, carla_location in enumerate(list_location):
- loc_array[i, 0] = carla_location.x
- loc_array[i, 1] = carla_location.y
- loc_array[i, 2] = carla_location.z
-
- return loc_array
-
-
-def list_wpt2array(list_wpt):
- """
- Convert list of carla transform to np.array
- Parameters
- ----------
- list_wpt : list
- List of carla waypoint.
-
- Returns
- -------
- loc_array : np.array
- Numpy array of shape (N, 3)
- """
- loc_array = np.zeros((len(list_wpt), 3))
- for i, carla_wpt in enumerate(list_wpt):
- loc_array[i, 0] = carla_wpt.transform.location.x
- loc_array[i, 1] = carla_wpt.transform.location.y
- loc_array[i, 2] = carla_wpt.transform.location.z
-
- return loc_array
-
-
-def convert_tl_status(status):
- """
- Convert carla.TrafficLightState to str.
- Parameters
- ----------
- status : carla.TrafficLightState
-
- Returns
- -------
- status_str : str
- """
- if status == carla.TrafficLightState.Red:
- return "red"
- elif status == carla.TrafficLightState.Green:
- return "green"
- elif status == carla.TrafficLightState.Yellow:
- return "yellow"
- else:
- return "normal"
-
-
-def x_to_world_transformation(transform):
- """
- Get the transformation matrix from x(it can be vehicle or sensor)
- coordinates to world coordinate.
-
- Parameters
- ----------
- transform : carla.Transform
- The transform that contains location and rotation
-
- Returns
- -------
- matrix : np.ndarray
- The transformation matrx.
-
- """
- rotation = transform.rotation
- location = transform.location
-
- # used for rotation matrix
- c_y = np.cos(np.radians(rotation.yaw))
- s_y = np.sin(np.radians(rotation.yaw))
- c_r = np.cos(np.radians(rotation.roll))
- s_r = np.sin(np.radians(rotation.roll))
- c_p = np.cos(np.radians(rotation.pitch))
- s_p = np.sin(np.radians(rotation.pitch))
-
- matrix = np.identity(4)
- # translation matrix
- matrix[0, 3] = location.x
- matrix[1, 3] = location.y
- matrix[2, 3] = location.z
-
- # rotation matrix
- matrix[0, 0] = c_p * c_y
- matrix[0, 1] = c_y * s_p * s_r - s_y * c_r
- matrix[0, 2] = -c_y * s_p * c_r - s_y * s_r
- matrix[1, 0] = s_y * c_p
- matrix[1, 1] = s_y * s_p * s_r + c_y * c_r
- matrix[1, 2] = -s_y * s_p * c_r + c_y * s_r
- matrix[2, 0] = s_p
- matrix[2, 1] = -c_p * s_r
- matrix[2, 2] = c_p * c_r
-
- return matrix
-
-
-def world_to_sensor(cords, sensor_transform):
- """
- Transform coordinates from world reference to sensor reference.
-
- Parameters
- ----------
- cords : np.ndarray
- Coordinates under world reference, shape: (4, n).
-
- sensor_transform : carla.Transform
- Sensor position in the world.
-
- Returns
- -------
- sensor_cords : np.ndarray
- Coordinates in the sensor reference.
-
- """
- sensor_world_matrix = x_to_world_transformation(sensor_transform)
- world_sensor_matrix = np.linalg.inv(sensor_world_matrix)
- sensor_cords = np.dot(world_sensor_matrix, cords)
-
- return sensor_cords
-
-
-def exclude_off_road_agents(static_bev, dynamic_bev):
- dynamic_bev[static_bev == 0] = 0
- return dynamic_bev
-
-
-def retrieve_city_object_info(world, label_list):
- """
- A general function to retrieve object bbx in carla world except vehicle,
- lane, crosswalk and road.
-
- Parameters
- ----------
- world : carla.World
- Carla world object
- label_list : list of str
- The label that users want to retrieve.
-
- Returns
- -------
- A dictionary with information of the retrieved objects.
- """
- city_object_info = {}
- for label_name in label_list:
- object_ins = world.get_level_bbs(LABEL_TO_CARLA[label_name])
-
- obj_info = {}
-
- for obj in object_ins:
- obj_id = uuid.uuid4().hex[:6].upper()
-
- obj_transform = carla.Transform(obj.location, obj.rotation)
- obj_loc = [
- obj.location.x,
- obj.location.y,
- obj.location.z,
- ]
- obj_yaw = obj.rotation.yaw
-
- # calculate 4 corners
- bb = obj.extent
- corners = [
- carla.Location(x=-bb.x, y=-bb.y),
- carla.Location(x=-bb.x, y=bb.y),
- carla.Location(x=bb.x, y=bb.y),
- carla.Location(x=bb.x, y=-bb.y),
- ]
-
- obj_transform.transform(corners)
- corners_reformat = [[x.x, x.y, x.z] for x in corners]
-
- obj_info[obj_id] = {"location": obj_loc, "yaw": obj_yaw, "corners": corners_reformat}
-
- city_object_info.update({label_name: obj_info})
-
- return city_object_info
-
-
-def obj_in_range(center, radius, obj_info_dict):
- """
- Retrieve the object in range.
-
- Parameters
- ----------
- center : carla.Transform
- The ego position
-
- radius : float
- Valid radius.
-
- obj_info_dict : dict
-
- Returns
- -------
- A dictionary that contains objects in range.
- """
- final_objs = {}
-
- for obj_category, obj_contents in obj_info_dict.items():
- cur_objs = {}
- for obj_id, obj_info in obj_contents.items():
- corners = obj_info["corners"]
- for corner in corners:
- distance = math.sqrt((corner[0] - center.location.x) ** 2 + (corner[1] - center.location.y) ** 2)
- if distance < radius:
- cur_objs.update({obj_id: obj_info})
- break
- final_objs.update({obj_category: cur_objs})
-
- return final_objs
diff --git a/OpenCOOD/logreplay/scenario/__init__.py b/OpenCOOD/logreplay/scenario/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/OpenCOOD/logreplay/scenario/scenarios_manager.py b/OpenCOOD/logreplay/scenario/scenarios_manager.py
deleted file mode 100644
index 5a2aa022..00000000
--- a/OpenCOOD/logreplay/scenario/scenarios_manager.py
+++ /dev/null
@@ -1,69 +0,0 @@
-import os
-from collections import OrderedDict
-
-from opencood.hypes_yaml.yaml_utils import load_yaml
-from logreplay.scenario.scene_manager import SceneManager
-
-
-class ScenariosManager:
- """
- Format all scenes in a structured way.
-
- Parameters
- ----------
- scenario_params: dict
- Overall parameters for the replayed scenes.
-
- Attributes
- ----------
-
- """
-
- def __init__(self, scenario_params):
- # this defines carla world sync mode, weather, town name, and seed.
- self.scene_params = scenario_params
-
- # e.g. /opv2v/data/train
- root_dir = self.scene_params["root_dir"]
-
- # first load all paths of different scenarios
- scenario_folders = sorted([os.path.join(root_dir, x) for x in os.listdir(root_dir) if os.path.isdir(os.path.join(root_dir, x))])
- self.scenario_database = OrderedDict()
-
- # loop over all scenarios
- for i, scenario_folder in enumerate(scenario_folders):
- scene_name = os.path.split(scenario_folder)[-1]
- self.scenario_database.update({scene_name: OrderedDict()})
-
- # load the collection yaml file
- protocol_yml = [x for x in os.listdir(scenario_folder) if x.endswith(".yaml")]
- collection_params = load_yaml(os.path.join(scenario_folder, protocol_yml[0]))
-
- # create the corresponding scene manager
- cur_sg = SceneManager(scenario_folder, scene_name, collection_params, scenario_params)
- self.scenario_database[scene_name].update({"scene_manager": cur_sg})
-
- def tick(self):
- """
- Tick for every scene manager to do the log replay.
- """
- for scene_name, scene_content in self.scenario_database.items():
- print("log replay %s" % scene_name)
- scene_manager = scene_content["scene_manager"]
- run_flag = True
-
- scene_manager.start_simulator()
-
- while run_flag:
- run_flag = scene_manager.tick()
-
- scene_manager.close()
-
-
-if __name__ == "__main__":
- from opencood.hypes_yaml.yaml_utils import load_yaml
-
- scene_params = load_yaml("../hypes_yaml/replay.yaml")
- scenarion_manager = ScenariosManager(scenario_params=scene_params)
- scenarion_manager.tick()
- print("test passed")
diff --git a/OpenCOOD/logreplay/scenario/scene_manager.py b/OpenCOOD/logreplay/scenario/scene_manager.py
deleted file mode 100644
index f49356f3..00000000
--- a/OpenCOOD/logreplay/scenario/scene_manager.py
+++ /dev/null
@@ -1,432 +0,0 @@
-import os
-import random
-import sys
-from collections import OrderedDict
-
-import carla
-import numpy as np
-
-from logreplay.assets.utils import find_town, find_blue_print
-from logreplay.assets.presave_lib import bcolors
-from logreplay.map.map_manager import MapManager
-from logreplay.sensors.sensor_manager import SensorManager
-from opencood.hypes_yaml.yaml_utils import load_yaml
-
-
-class SceneManager:
- """
- Manager for each scene for spawning, moving and destroying.
-
- Parameters
- ----------
- folder : str
- The folder to the current scene.
-
- scene_name : str
- The scene's name.
-
- collection_params : dict
- The collecting protocol information.
-
- scenario_params : dict
- The replay params.
- """
-
- def __init__(self, folder, scene_name, collection_params, scenario_params):
- self.scene_name = scene_name
- self.town_name = find_town(scene_name)
- self.collection_params = collection_params
- self.scenario_params = scenario_params
- self.cav_id_list = []
-
- # dumping related
- self.output_root = os.path.join(scenario_params["output_dir"], scene_name)
-
- if "seed" in collection_params["world"]:
- np.random.seed(collection_params["world"]["seed"])
- random.seed(collection_params["world"]["seed"])
-
- # at least 1 cav should show up
- cav_list = sorted([x for x in os.listdir(folder) if os.path.isdir(os.path.join(folder, x))])
- assert len(cav_list) > 0
-
- self.database = OrderedDict()
- # we want to save timestamp as the parent keys for cavs
- cav_sample = os.path.join(folder, cav_list[0])
-
- yaml_files = sorted([os.path.join(cav_sample, x) for x in os.listdir(cav_sample) if x.endswith(".yaml") and "additional" not in x])
- self.timestamps = self.extract_timestamps(yaml_files)
-
- # loop over all timestamps
- for timestamp in self.timestamps:
- self.database[timestamp] = OrderedDict()
- # loop over all cavs
- for j, cav_id in enumerate(cav_list):
- if cav_id not in self.cav_id_list:
- self.cav_id_list.append(str(cav_id))
-
- self.database[timestamp][cav_id] = OrderedDict()
- cav_path = os.path.join(folder, cav_id)
-
- yaml_file = os.path.join(cav_path, timestamp + ".yaml")
- self.database[timestamp][cav_id]["yaml"] = yaml_file
-
- # this is used to dynamically save all information of the objects
- self.veh_dict = OrderedDict()
- # used to count timestamp
- self.cur_count = 0
- # used for HDMap
- self.map_manager = None
-
- def start_simulator(self):
- """
- Connect to the carla simulator for log replay.
- """
- simulation_config = self.collection_params["world"]
-
- # simulation sync mode time step
- fixed_delta_seconds = simulation_config["fixed_delta_seconds"]
- weather_config = simulation_config["weather"] if "weather" in simulation_config else None
-
- # setup the carla client
- self.client = carla.Client("localhost", simulation_config["client_port"])
- self.client.set_timeout(10.0)
-
- # load the map
- if self.town_name != "Culver_City":
- try:
- self.world = self.client.load_world(self.town_name)
- except RuntimeError:
- print(
- f"{bcolors.FAIL} %s is not found in your CARLA repo! "
- f"Please download all town maps to your CARLA "
- f"repo!{bcolors.ENDC}" % self.town_name
- )
- else:
- self.world = self.client.get_world()
-
- if not self.world:
- sys.exit("World loading failed")
-
- # setup the new setting
- self.origin_settings = self.world.get_settings()
- new_settings = self.world.get_settings()
-
- new_settings.synchronous_mode = True # noqa: DC05
- new_settings.fixed_delta_seconds = fixed_delta_seconds
-
- self.world.apply_settings(new_settings)
- # set weather if needed
- if weather_config is not None:
- weather = self.set_weather(weather_config)
- self.world.set_weather(weather)
- # get map
- self.carla_map = self.world.get_map()
- # spectator
- self.spectator = self.world.get_spectator()
- # hd map manager per scene
- self.map_manager = MapManager(self.world, self.scenario_params["map"], self.output_root, self.scene_name)
-
- def tick(self):
- """
- Spawn the vehicle to the correct position
- """
- if self.cur_count >= len(self.timestamps):
- return False
-
- cur_timestamp = self.timestamps[self.cur_count]
- cur_database = self.database[cur_timestamp]
-
- for i, (cav_id, cav_yml) in enumerate(cur_database.items()):
- cav_content = load_yaml(cav_yml["yaml"])
- if cav_id not in self.veh_dict:
- self.spawn_cav(cav_id, cav_content, cur_timestamp)
- else:
- self.move_vehicle(cav_id, cur_timestamp, self.structure_transform_cav((cav_content["true_ego_pos"])))
-
- self.veh_dict[cav_id]["cav"] = True
- # spawn the sensor on each cav
- if "sensor_manager" not in self.veh_dict[cav_id]:
- self.veh_dict[cav_id]["sensor_manager"] = SensorManager(
- cav_id, self.veh_dict[cav_id], self.world, self.scenario_params["sensor"], self.output_root
- )
-
- # set the spectator to the first cav
- if i == 0:
- transform = self.structure_transform_cav(cav_content["true_ego_pos"])
- self.spectator.set_transform(carla.Transform(transform.location + carla.Location(z=70), carla.Rotation(pitch=-90)))
-
- for bg_veh_id, bg_veh_content in cav_content["vehicles"].items():
- if str(bg_veh_id) not in self.veh_dict:
- self.spawn_bg_vehicles(bg_veh_id, bg_veh_content, cur_timestamp)
- else:
- self.move_vehicle(
- str(bg_veh_id), cur_timestamp, self.structure_transform_bg_veh(bg_veh_content["location"], bg_veh_content["angle"])
- )
- # remove the vehicles that are not in any cav's scope
- self.destroy_vehicle(cur_timestamp)
-
- self.cur_count += 1
- self.world.tick()
-
- # we dump data after tick() so the agent can retrieve the newest info
- self.sensor_dumping(cur_timestamp)
- self.map_dumping()
-
- return True
-
- def map_dumping(self):
- """
- Dump bev map related.
-
- Parameters
- ----------
- """
- for veh_id, veh_content in self.veh_dict.items():
- if "cav" in veh_content:
- self.map_manager.run_step(veh_id, veh_content, self.veh_dict)
-
- def sensor_dumping(self, cur_timestamp):
- for veh_id, veh_content in self.veh_dict.items():
- if "sensor_manager" in veh_content:
- veh_content["sensor_manager"].run_step(cur_timestamp)
-
- def spawn_cav(self, cav_id, cav_content, cur_timestamp):
- """
- Spawn the cav based on current content.
-
- Parameters
- ----------
- cav_id : str
- The saved cav_id.
-
- cav_content : dict
- The information in the cav's folder.
-
- cur_timestamp : str
- This is used to judge whether this vehicle has been already
- called in this timestamp.
- """
-
- # cav always use lincoln
- model = "vehicle.lincoln.mkz_2017"
-
- # retrive the blueprint library
- blueprint_library = self.world.get_blueprint_library()
- cav_bp = blueprint_library.find(model)
- # cav is always green
- color = "0, 0, 255"
- cav_bp.set_attribute("color", color)
-
- cur_pose = cav_content["true_ego_pos"]
- # convert to carla needed format
- cur_pose = self.structure_transform_cav(cur_pose)
-
- # spawn the vehicle
- vehicle = self.world.try_spawn_actor(cav_bp, cur_pose)
-
- while not vehicle:
- cur_pose.location.z += 0.01
- vehicle = self.world.try_spawn_actor(cav_bp, cur_pose)
-
- self.veh_dict.update(
- {
- str(cav_id): {
- "cur_pose": cur_pose,
- "model": model,
- "color": color,
- "actor_id": vehicle.id,
- "actor": vehicle,
- "cur_count": cur_timestamp,
- }
- }
- )
-
- def spawn_bg_vehicles(self, bg_veh_id, bg_veh_content, cur_timestamp):
- """
- Spawn the background vehicle.
-
- Parameters
- ----------
- bg_veh_id : str
- The id of the bg vehicle.
- bg_veh_content : dict
- The contents of the bg vehicle
- cur_timestamp : str
- This is used to judge whether this vehicle has been already
- called in this timestamp.
- """
- # retrieve the blueprint library
- blueprint_library = self.world.get_blueprint_library()
-
- cur_pose = self.structure_transform_bg_veh(bg_veh_content["location"], bg_veh_content["angle"])
- if str(bg_veh_id) in self.cav_id_list:
- model = "vehicle.lincoln.mkz_2017"
- veh_bp = blueprint_library.find(model)
- color = "0, 0, 255"
- else:
- model = find_blue_print(bg_veh_content["extent"])
- if not model:
- print("model net found for %s" % bg_veh_id)
- veh_bp = blueprint_library.find(model)
-
- color = random.choice(veh_bp.get_attribute("color").recommended_values)
-
- veh_bp.set_attribute("color", color)
-
- # spawn the vehicle
- vehicle = self.world.try_spawn_actor(veh_bp, cur_pose)
-
- while not vehicle:
- cur_pose.location.z += 0.01
- vehicle = self.world.try_spawn_actor(veh_bp, cur_pose)
-
- self.veh_dict.update(
- {
- str(bg_veh_id): {
- "cur_pose": cur_pose,
- "model": model,
- "color": color,
- "actor_id": vehicle.id,
- "actor": vehicle,
- "cur_count": cur_timestamp,
- }
- }
- )
-
- def move_vehicle(self, veh_id, cur_timestamp, transform):
- """
- Move the
-
- Parameters
- ----------
- veh_id : str
- Vehicle's id
-
- cur_timestamp : str
- Current timestamp
-
- transform : carla.Transform
- Current pose/
- """
- # this represent this vehicle is already moved in this round
- if self.veh_dict[veh_id]["cur_count"] == cur_timestamp:
- return
-
- self.veh_dict[veh_id]["actor"].set_transform(transform)
- self.veh_dict[veh_id]["cur_count"] = cur_timestamp
- self.veh_dict[veh_id]["cur_pose"] = transform
-
- def close(self):
- self.world.apply_settings(self.origin_settings)
- actor_list = self.world.get_actors()
- for actor in actor_list:
- actor.destroy()
- self.map_manager.destroy()
- self.sensor_destory()
-
- def sensor_destory(self):
- for veh_id, veh_content in self.veh_dict.items():
- if "sensor_manager" in veh_content:
- veh_content["sensor_manager"].destroy()
-
- def destroy_vehicle(self, cur_timestamp):
- """
- This is used to destroy the vehicles that are out of any cav's scope.
- """
- destroy_list = []
- for veh_id, veh_content in self.veh_dict.items():
- if veh_content["cur_count"] != cur_timestamp:
- veh_content["actor"].destroy()
- destroy_list.append(veh_id)
-
- for veh_id in destroy_list:
- self.veh_dict.pop(veh_id)
-
- def structure_transform_cav(self, pose):
- """
- Convert the pose saved in list to transform format.
-
- Parameters
- ----------
- pose : list
- x, y, z, roll, yaw, pitch
-
- Returns
- -------
- carla.Transform
- """
- cur_pose = carla.Transform(carla.Location(x=pose[0], y=pose[1], z=pose[2]), carla.Rotation(roll=pose[3], yaw=pose[4], pitch=pose[5]))
-
- return cur_pose
-
- @staticmethod
- def structure_transform_bg_veh(location, rotation):
- """
- Convert the location and rotation in list to carla transform format.
-
- Parameters
- ----------
- location : list
- rotation : list
- Returns
- -------
- carla.Transform
- """
- cur_pose = carla.Transform(
- carla.Location(x=location[0], y=location[1], z=location[2]), carla.Rotation(roll=rotation[0], yaw=rotation[1], pitch=rotation[2])
- )
-
- return cur_pose
-
- @staticmethod
- def extract_timestamps(yaml_files):
- """
- Given the list of the yaml files, extract the mocked timestamps.
-
- Parameters
- ----------
- yaml_files : list
- The full path of all yaml files of ego vehicle
-
- Returns
- -------
- timestamps : list
- The list containing timestamps only.
- """
- timestamps = []
-
- for file in yaml_files:
- res = file.split("/")[-1]
-
- timestamp = res.replace(".yaml", "")
- timestamps.append(timestamp)
-
- return timestamps
-
- @staticmethod
- def set_weather(weather_settings):
- """
- Set CARLA weather params.
-
- Parameters
- ----------
- weather_settings : dict
- The dictionary that contains all parameters of weather.
-
- Returns
- -------
- The CARLA weather setting.
- """
- weather = carla.WeatherParameters(
- sun_altitude_angle=weather_settings["sun_altitude_angle"],
- cloudiness=weather_settings["cloudiness"],
- precipitation=weather_settings["precipitation"],
- precipitation_deposits=weather_settings["precipitation_deposits"],
- wind_intensity=weather_settings["wind_intensity"],
- fog_density=weather_settings["fog_density"],
- fog_distance=weather_settings["fog_distance"],
- fog_falloff=weather_settings["fog_falloff"],
- wetness=weather_settings["wetness"],
- )
- return weather
diff --git a/OpenCOOD/logreplay/sensors/__init__.py b/OpenCOOD/logreplay/sensors/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/OpenCOOD/logreplay/sensors/base_sensor.py b/OpenCOOD/logreplay/sensors/base_sensor.py
deleted file mode 100644
index 95be66f2..00000000
--- a/OpenCOOD/logreplay/sensors/base_sensor.py
+++ /dev/null
@@ -1,23 +0,0 @@
-"""
-Base Class for sensors
-"""
-
-import cv2
-
-
-class BaseSensor:
- def __init__(self, agent_id, vehicle, world, config, global_position):
- return
-
- def visualize_data(self):
- return
-
- def data_dump(self, output_folder, cur_timestamp):
- return
-
- def tick(self):
- return None
-
- def destroy(self):
- self.sensor.destroy()
- cv2.destroyAllWindows()
diff --git a/OpenCOOD/logreplay/sensors/bev_semantic_camera.py b/OpenCOOD/logreplay/sensors/bev_semantic_camera.py
deleted file mode 100644
index d1bdaaed..00000000
--- a/OpenCOOD/logreplay/sensors/bev_semantic_camera.py
+++ /dev/null
@@ -1,197 +0,0 @@
-"""
-CARLA Semantic Camera Sensor
-"""
-
-import os.path
-import weakref
-
-import carla
-import cv2
-import numpy as np
-
-from opencood.hypes_yaml.yaml_utils import save_yaml_wo_overwriting
-from logreplay.sensors.utils import get_camera_intrinsic
-from logreplay.sensors.base_sensor import BaseSensor
-
-
-class BEVSemanticCamera(BaseSensor):
- """
- BEV camera for semantic segmentation.
-
- Parameters
- ----------
- vehicle : carla.Vehicle
- The carla.Vehicle, this is for cav.
-
- world : carla.World
- The carla world object, this is for rsu.
-
- config : dict
- Semantic camera configuration.
-
- global_position : list
- Global position of the infrastructure, [x, y, z]
-
- Attributes
- ----------
- image : np.ndarray
- Current received rgb image.
- sensor : carla.sensor
- The carla sensor that mounts at the vehicle.
-
- """
-
- def __init__(self, agent_id, vehicle, world, config, global_position):
- super().__init__(agent_id, vehicle, world, config, global_position)
- if vehicle is not None:
- world = vehicle.get_world()
-
- self.agent_id = agent_id
- self.name = "bev_semantic_camera"
-
- blueprint = world.get_blueprint_library().find("sensor.camera.semantic_segmentation")
- blueprint.set_attribute("fov", str(config["fov"]))
- blueprint.set_attribute("image_size_x", str(config["image_size_x"]))
- blueprint.set_attribute("image_size_y", str(config["image_size_y"]))
- self.height = config["height"]
- self.visualize = config["visualize"]
-
- spawn_point = self.spawn_point_estimation(global_position)
-
- if vehicle is not None:
- self.sensor = world.spawn_actor(blueprint, spawn_point, attach_to=vehicle)
- else:
- self.sensor = world.spawn_actor(blueprint, spawn_point)
-
- self.image = None
- self.timstamp = None # noqa: DC05
- self.frame = 0
- weak_self = weakref.ref(self)
- self.sensor.listen(lambda event: BEVSemanticCamera._on_rgb_image_event(weak_self, event))
-
- # camera attributes
- self.image_width = int(self.sensor.attributes["image_size_x"])
- self.image_height = int(self.sensor.attributes["image_size_y"])
-
- def spawn_point_estimation(self, global_position):
- pitch = -90
- carla_location = carla.Location(x=0, y=0, z=self.height)
-
- if global_position is not None:
- carla_location = carla.Location(x=global_position[0], y=global_position[1], z=self.height)
-
- carla_rotation = carla.Rotation(roll=0, yaw=0, pitch=pitch)
- spawn_point = carla.Transform(carla_location, carla_rotation)
-
- return spawn_point
-
- @staticmethod
- def labels_to_array(bgr_image):
- """
- Convert an image containing CARLA semantic segmentation labels to a
- 2D array containing the label of each pixel.
-
- Parameters
- ----------
- bgr_image : np.ndarray
- BGR image
- """
- return bgr_image[:, :, 2]
-
- @staticmethod
- def labels_to_cityscapes_palette(label):
- """
- Convert an image containing CARLA semantic segmentation labels to
- Cityscapes palette.
-
- Parameters
- ----------
- label : np.ndarray
- label image, (h, w)
-
- Returns
- -------
- Converted BGR image.
- """
- classes = {
- 0: [0, 0, 0], # None
- 1: [70, 70, 70], # Buildings
- 2: [190, 153, 153], # Fences
- 3: [72, 0, 90], # Other
- 4: [220, 20, 60], # Pedestrians
- 5: [153, 153, 153], # Poles
- 6: [157, 234, 50], # RoadLines
- 7: [128, 64, 128], # Roads
- 8: [244, 35, 232], # Sidewalks
- 9: [107, 142, 35], # Vegetation
- 10: [0, 0, 255], # Vehicles
- 11: [102, 102, 156], # Walls
- 12: [220, 220, 0], # TrafficSigns
- 13: [70, 130, 180], # Sky
- 14: [81, 0, 81], # Ground
- 15: [150, 100, 100], # Bridge
- 16: [230, 150, 140], # RailTrack
- 17: [180, 165, 180], # All types of guard rails/crash barriers.
- 18: [250, 170, 30], # Traffic Light
- 19: [110, 190, 160], # Static
- 20: [170, 120, 50], # Dynamic
- 21: [45, 60, 150], # Water
- 22: [145, 170, 100], # Terrain
- }
- result = np.zeros((label.shape[0], label.shape[1], 3), dtype=np.uint8)
- for key, value in classes.items():
- result[np.where(label == key)] = value
- return result
-
- @staticmethod
- def _on_rgb_image_event(weak_self, event):
- """CAMERA method"""
- self = weak_self()
- if not self:
- return
- image = np.array(event.raw_data)
- image = image.reshape((self.image_height, self.image_width, 4))
- # we need to remove the alpha channel
- image = image[:, :, :3]
- # self.image is the dumped data
- self.image = np.array(self.labels_to_array(image), dtype=int)
- self.vis_image = cv2.cvtColor(self.labels_to_cityscapes_palette(self.image), cv2.COLOR_BGR2RGB)
-
- self.frame = event.frame
- self.timestamp = event.timestamp
-
- def visualize_data(self):
- if self.visualize:
- while not hasattr(self, "vis_image") or self.vis_image is None:
- continue
- cv2.imshow("bev seg camera agent %s" % self.agent_id, self.vis_image)
- cv2.waitKey(1)
-
- def data_dump(self, output_root, cur_timestamp):
- while not hasattr(self, "vis_image") or self.vis_image is None:
- continue
- # dump visualization
- output_vis_name = os.path.join(output_root, cur_timestamp + "_bev_sem_vis.png")
- cv2.imwrite(output_vis_name, self.vis_image)
-
- # dump the label
- output_label_name = os.path.join(output_root, cur_timestamp + "_bev_sem_label.png")
- cv2.imwrite(output_label_name, self.image)
-
- # dump the yaml
- save_yaml_name = os.path.join(output_root, cur_timestamp + "_additional.yaml")
- # intrinsic
- camera_intrinsic = get_camera_intrinsic(self.sensor)
- # pose under world coordinate system
- camera_transformation = self.sensor.get_transform()
- cords = [
- camera_transformation.location.x,
- camera_transformation.location.y,
- camera_transformation.location.z,
- camera_transformation.rotation.roll,
- camera_transformation.rotation.yaw,
- camera_transformation.rotation.pitch,
- ]
-
- bev_sem_cam_info = {"bev_sem_cam": {"intrinsic": camera_intrinsic, "cords": cords}}
- save_yaml_wo_overwriting(bev_sem_cam_info, save_yaml_name)
diff --git a/OpenCOOD/logreplay/sensors/semantic_lidar.py b/OpenCOOD/logreplay/sensors/semantic_lidar.py
deleted file mode 100644
index 0d6bc4a9..00000000
--- a/OpenCOOD/logreplay/sensors/semantic_lidar.py
+++ /dev/null
@@ -1,120 +0,0 @@
-"""
-This is mainly used to filter out objects that is not in the sight
-of cameras.
-"""
-
-import weakref
-
-import carla
-import numpy as np
-from logreplay.sensors.base_sensor import BaseSensor
-
-
-class SemanticLidar(BaseSensor):
- def __init__(self, agent_id, vehicle, world, config, global_position):
- super().__init__(agent_id, vehicle, world, config, global_position)
-
- if vehicle is not None:
- world = vehicle.get_world()
-
- self.agent_id = agent_id
-
- blueprint = world.get_blueprint_library().find("sensor.lidar.ray_cast_semantic")
- # set attribute based on the configuration
- blueprint.set_attribute("upper_fov", str(config["upper_fov"]))
- blueprint.set_attribute("lower_fov", str(config["lower_fov"]))
- blueprint.set_attribute("channels", str(config["channels"]))
- blueprint.set_attribute("range", str(config["range"]))
- blueprint.set_attribute("points_per_second", str(config["points_per_second"]))
- blueprint.set_attribute("rotation_frequency", str(config["rotation_frequency"]))
-
- relative_position = config["relative_pose"]
- spawn_point = self.spawn_point_estimation(relative_position, global_position)
- self.name = "semantic_lidar" + str(relative_position)
- self.thresh = config["thresh"]
-
- if vehicle is not None:
- self.sensor = world.spawn_actor(blueprint, spawn_point, attach_to=vehicle)
- else:
- self.sensor = world.spawn_actor(blueprint, spawn_point)
-
- # lidar data
- self.points = None
- self.obj_idx = None
- self.obj_tag = None
-
- self.timestamp = None
- self.frame = 0
-
- weak_self = weakref.ref(self)
- self.sensor.listen(lambda event: SemanticLidar._on_data_event(weak_self, event))
-
- @staticmethod
- def _on_data_event(weak_self, event):
- """Semantic Lidar method"""
- self = weak_self()
- if not self:
- return
-
- # shape:(n, 6)
- data = np.frombuffer(
- event.raw_data,
- dtype=np.dtype(
- [("x", np.float32), ("y", np.float32), ("z", np.float32), ("CosAngle", np.float32), ("ObjIdx", np.uint32), ("ObjTag", np.uint32)]
- ),
- )
-
- # (x, y, z, intensity)
- self.points = np.array([data["x"], data["y"], data["z"]]).T
- self.obj_tag = np.array(data["ObjTag"])
- self.obj_idx = np.array(data["ObjIdx"])
-
- self.data = data
- self.frame = event.frame
- self.timestamp = event.timestamp
-
- @staticmethod
- def spawn_point_estimation(relative_position, global_position):
- pitch = 0
- carla_location = carla.Location(x=0, y=0, z=0)
-
- if global_position is not None:
- carla_location = carla.Location(x=global_position[0], y=global_position[1], z=global_position[2])
- pitch = -35
-
- if relative_position == "front":
- carla_location = carla.Location(x=carla_location.x + 2.5, y=carla_location.y, z=carla_location.z + 1.0)
- yaw = 0
-
- elif relative_position == "right":
- carla_location = carla.Location(x=carla_location.x + 0.0, y=carla_location.y + 0.3, z=carla_location.z + 1.8)
- yaw = 100
-
- elif relative_position == "left":
- carla_location = carla.Location(x=carla_location.x + 0.0, y=carla_location.y - 0.3, z=carla_location.z + 1.8)
- yaw = -100
- else:
- carla_location = carla.Location(x=carla_location.x - 2.0, y=carla_location.y, z=carla_location.z + 1.5)
- yaw = 180
-
- carla_rotation = carla.Rotation(roll=0, yaw=yaw, pitch=pitch)
- spawn_point = carla.Transform(carla_location, carla_rotation)
-
- return spawn_point
-
- def tick(self):
- while self.obj_idx is None or self.obj_tag is None or self.obj_idx.shape[0] != self.obj_tag.shape[0]:
- continue
-
- # label 10 is the vehicle
- vehicle_idx = self.obj_idx[self.obj_tag == 10]
- # each individual instance id
- vehicle_unique_id = list(np.unique(vehicle_idx))
- vehicle_id_filter = []
-
- for veh_id in vehicle_unique_id:
- if vehicle_idx[vehicle_idx == veh_id].shape[0] > self.thresh:
- vehicle_id_filter.append(veh_id)
-
- # these are the ids that are visible
- return vehicle_id_filter
diff --git a/OpenCOOD/logreplay/sensors/sensor_manager.py b/OpenCOOD/logreplay/sensors/sensor_manager.py
deleted file mode 100644
index 5eb39973..00000000
--- a/OpenCOOD/logreplay/sensors/sensor_manager.py
+++ /dev/null
@@ -1,82 +0,0 @@
-"""
-Sensor Manager for each cav
-"""
-
-import importlib
-import os
-from collections import OrderedDict
-
-
-class SensorManager:
- """
- The manager controls all sensor data streaming and dumping for each cav.
- todo: rsu not considered yet.
- Parameters
- ----------
- agent_id : str
- The cav's original id.
-
- vehicle_content : dict
- The content of the cav.
-
- world : carla.World
- Carla simulation server object.
-
- config_yaml : dict
- Configurations for the sensor manager.
-
- output_root : str
- Output directory for data dumping.
-
- Attributes
- ----------
- sensor_list : list
- A list of sensors to dump/visualize data.
- """
-
- def __init__(self, agent_id, vehicle_content, world, config_yaml, output_root):
- self.agent_id = agent_id
- self.output_root = output_root
- self.vehicle = vehicle_content["actor"]
- self.world = world
- self.sensor_list = []
- # this is used to gather the meta information return from sensors
- self.sensor_meta = OrderedDict()
-
- for sensor_content in config_yaml["sensor_list"]:
- sensor = None
- sensor_name = sensor_content["name"]
-
- # find the relative path
- sensor_filename = "logreplay.sensors." + sensor_name
- sensor_lib = importlib.import_module(sensor_filename)
- target_sensor_name = sensor_name.replace("_", "")
-
- # the sensor corresponding class has to have the same
- # name pattern as the file
- for name, cls in sensor_lib.__dict__.items():
- if name.lower() == target_sensor_name.lower():
- sensor = cls
-
- assert sensor is not None, "The sensor class name has to be thesame as the file name. e.g. bev_semantic_camera -> BevSemanticCamera"
- # todo: rsu is not considered yet
- sensor_instance = sensor(self.agent_id, self.vehicle, self.world, sensor_content["args"], None)
- self.sensor_list.append(sensor_instance)
-
- def run_step(self, cur_timestamp):
- for sensor_instance in self.sensor_list:
- sensor_name = sensor_instance.name
- sensor_instance.visualize_data()
-
- meta_info = sensor_instance.tick()
- self.sensor_meta.update({sensor_name: meta_info})
-
- # for data dumping
- output_folder = os.path.join(self.output_root, self.agent_id)
- if not os.path.exists(output_folder):
- os.makedirs(output_folder)
- sensor_instance.data_dump(output_folder, cur_timestamp)
-
- def destroy(self):
- for sensor_instance in self.sensor_list:
- sensor_instance.destroy()
diff --git a/OpenCOOD/logreplay/sensors/utils.py b/OpenCOOD/logreplay/sensors/utils.py
deleted file mode 100644
index a729d178..00000000
--- a/OpenCOOD/logreplay/sensors/utils.py
+++ /dev/null
@@ -1,28 +0,0 @@
-import numpy as np
-
-
-def get_camera_intrinsic(sensor):
- """
- Retrieve the camera intrinsic matrix.
-
- Parameters
- ----------
- sensor : carla.sensor
- Carla rgb camera object.
-
- Returns
- -------
- matrix_x : np.ndarray
- The 2d intrinsic matrix.
-
- """
- VIEW_WIDTH = int(sensor.attributes["image_size_x"])
- VIEW_HEIGHT = int(sensor.attributes["image_size_y"])
- VIEW_FOV = int(float(sensor.attributes["fov"]))
-
- matrix_k = np.identity(3)
- matrix_k[0, 2] = VIEW_WIDTH / 2.0
- matrix_k[1, 2] = VIEW_HEIGHT / 2.0
- matrix_k[0, 0] = matrix_k[1, 1] = VIEW_WIDTH / (2.0 * np.tan(VIEW_FOV * np.pi / 360.0))
-
- return matrix_k
diff --git a/OpenCOOD/opencood/__init__.py b/OpenCOOD/opencood/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/OpenCOOD/opencood/data_utils/__init__.py b/OpenCOOD/opencood/data_utils/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/OpenCOOD/opencood/data_utils/augmentor/__init__.py b/OpenCOOD/opencood/data_utils/augmentor/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/OpenCOOD/opencood/data_utils/augmentor/augment_utils.py b/OpenCOOD/opencood/data_utils/augmentor/augment_utils.py
deleted file mode 100644
index 9ed27d08..00000000
--- a/OpenCOOD/opencood/data_utils/augmentor/augment_utils.py
+++ /dev/null
@@ -1,42 +0,0 @@
-import numpy as np
-from opencood.utils import common_utils
-
-
-def global_rotation(gt_boxes, points, rot_range):
- """
- Args:
- gt_boxes: (N, 7 + C), [x, y, z, dx, dy, dz, heading, [vx], [vy]]
- points: (M, 3 + C),
- rot_range: [min, max]
- Returns:
- """
- noise_rotation = np.random.uniform(rot_range[0], rot_range[1])
- points = common_utils.rotate_points_along_z(points[np.newaxis, :, :], np.array([noise_rotation]))[0]
-
- gt_boxes[:, 0:3] = common_utils.rotate_points_along_z(gt_boxes[np.newaxis, :, 0:3], np.array([noise_rotation]))[0]
- gt_boxes[:, 6] += noise_rotation
-
- if gt_boxes.shape[1] > 7:
- gt_boxes[:, 7:9] = common_utils.rotate_points_along_z(
- np.hstack((gt_boxes[:, 7:9], np.zeros((gt_boxes.shape[0], 1))))[np.newaxis, :, :],
- np.array([noise_rotation]),
- )[0][:, 0:2]
-
- return gt_boxes, points
-
-
-def global_scaling(gt_boxes, points, scale_range):
- """
- Args:
- gt_boxes: (N, 7), [x, y, z, dx, dy, dz, heading]
- points: (M, 3 + C),
- scale_range: [min, max]
- Returns:
- """
- if scale_range[1] - scale_range[0] < 1e-3:
- return gt_boxes, points
- noise_scale = np.random.uniform(scale_range[0], scale_range[1])
- points[:, :3] *= noise_scale
- gt_boxes[:, :6] *= noise_scale
-
- return gt_boxes, points
diff --git a/OpenCOOD/opencood/data_utils/augmentor/data_augmentor.py b/OpenCOOD/opencood/data_utils/augmentor/data_augmentor.py
deleted file mode 100644
index 99adb36a..00000000
--- a/OpenCOOD/opencood/data_utils/augmentor/data_augmentor.py
+++ /dev/null
@@ -1,116 +0,0 @@
-"""
-Class for data augmentation
-"""
-
-from functools import partial
-from opencood.data_utils.augmentor import augment_utils
-
-
-class DataAugmentor(object):
- """
- Data Augmentor.
-
- Parameters
- ----------
- augment_config : list
- A list of augmentation configuration.
-
- Attributes
- ----------
- data_augmentor_queue : list
- The list of data augmented functions.
- """
-
- def __init__(self, augment_config, train=True):
- self.data_augmentor_queue = []
- self.train = train
-
- for cur_cfg in augment_config:
- cur_augmentor = getattr(self, cur_cfg["NAME"])(config=cur_cfg)
- self.data_augmentor_queue.append(cur_augmentor)
-
- def random_world_flip(self, data_dict=None, config=None):
- if data_dict is None:
- return partial(self.random_world_flip, config=config)
-
- gt_boxes, gt_mask, points = (
- data_dict["object_bbx_center"],
- data_dict["object_bbx_mask"],
- data_dict["lidar_np"],
- )
- gt_boxes_valid = gt_boxes[gt_mask == 1]
-
- for cur_axis in config["ALONG_AXIS_LIST"]:
- assert cur_axis in ["x", "y"]
- gt_boxes_valid, points = getattr(augment_utils, "random_flip_along_%s" % cur_axis)(
- gt_boxes_valid,
- points,
- )
-
- gt_boxes[: gt_boxes_valid.shape[0], :] = gt_boxes_valid
-
- data_dict["object_bbx_center"] = gt_boxes
- data_dict["object_bbx_mask"] = gt_mask
- data_dict["lidar_np"] = points
-
- return data_dict
-
- def random_world_rotation(self, data_dict=None, config=None):
- if data_dict is None:
- return partial(self.random_world_rotation, config=config)
-
- rot_range = config["WORLD_ROT_ANGLE"]
- if not isinstance(rot_range, list):
- rot_range = [-rot_range, rot_range]
-
- gt_boxes, gt_mask, points = (
- data_dict["object_bbx_center"],
- data_dict["object_bbx_mask"],
- data_dict["lidar_np"],
- )
- gt_boxes_valid = gt_boxes[gt_mask == 1]
- gt_boxes_valid, points = augment_utils.global_rotation(gt_boxes_valid, points, rot_range=rot_range)
- gt_boxes[: gt_boxes_valid.shape[0], :] = gt_boxes_valid
-
- data_dict["object_bbx_center"] = gt_boxes
- data_dict["object_bbx_mask"] = gt_mask
- data_dict["lidar_np"] = points
-
- return data_dict
-
- def random_world_scaling(self, data_dict=None, config=None):
- if data_dict is None:
- return partial(self.random_world_scaling, config=config)
-
- gt_boxes, gt_mask, points = (
- data_dict["object_bbx_center"],
- data_dict["object_bbx_mask"],
- data_dict["lidar_np"],
- )
- gt_boxes_valid = gt_boxes[gt_mask == 1]
-
- gt_boxes_valid, points = augment_utils.global_scaling(gt_boxes_valid, points, config["WORLD_SCALE_RANGE"])
- gt_boxes[: gt_boxes_valid.shape[0], :] = gt_boxes_valid
-
- data_dict["object_bbx_center"] = gt_boxes
- data_dict["object_bbx_mask"] = gt_mask
- data_dict["lidar_np"] = points
-
- return data_dict
-
- def forward(self, data_dict):
- """
- Args:
- data_dict:
- points: (N, 3 + C_in)
- gt_boxes: optional, (N, 7) [x, y, z, dx, dy, dz, heading]
- gt_names: optional, (N), string
- ...
-
- Returns:
- """
- if self.train:
- for cur_augmentor in self.data_augmentor_queue:
- data_dict = cur_augmentor(data_dict=data_dict)
-
- return data_dict
diff --git a/OpenCOOD/opencood/data_utils/datasets/__init__.py b/OpenCOOD/opencood/data_utils/datasets/__init__.py
deleted file mode 100644
index d9300026..00000000
--- a/OpenCOOD/opencood/data_utils/datasets/__init__.py
+++ /dev/null
@@ -1,50 +0,0 @@
-from importlib import import_module
-
-
-_DATASET_REGISTRY = {
- "LateFusionDataset": (
- "opencood.data_utils.datasets.late_fusion_dataset",
- "LateFusionDataset",
- ),
- "EarlyFusionDataset": (
- "opencood.data_utils.datasets.early_fusion_dataset",
- "EarlyFusionDataset",
- ),
- "IntermediateFusionDataset": (
- "opencood.data_utils.datasets.intermediate_fusion_dataset",
- "IntermediateFusionDataset",
- ),
- "IntermediateFusionDatasetV2": (
- "opencood.data_utils.datasets.intermediate_fusion_dataset_v2",
- "IntermediateFusionDatasetV2",
- ),
-}
-
-__all__ = tuple(_DATASET_REGISTRY)
-
-# the final range for evaluation
-GT_RANGE = [-140.8, -41.6, -10.6, 140.8, 41.6, 10.6]
-# The communication range for cavs
-COM_RANGE = 70
-
-
-def _load_dataset_class(dataset_name):
- module_name, class_name = _DATASET_REGISTRY[dataset_name]
- dataset_class = getattr(import_module(module_name), class_name)
- globals()[dataset_name] = dataset_class
- return dataset_class
-
-
-def __getattr__(name): # Noqa: DC02
- if name not in _DATASET_REGISTRY:
- raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
- return _load_dataset_class(name)
-
-
-def build_dataset(dataset_cfg, visualize=False, train=True, payload_handler=None):
- dataset_name = dataset_cfg["fusion"]["core_method"]
- error_message = f"{dataset_name} is not found. Please add the dataset to opencood.data_utils.datasets"
- assert dataset_name in _DATASET_REGISTRY, error_message
-
- dataset_class = _load_dataset_class(dataset_name)
- return dataset_class(params=dataset_cfg, visualize=visualize, train=train, payload_handler=payload_handler)
diff --git a/OpenCOOD/opencood/data_utils/datasets/basedataset.py b/OpenCOOD/opencood/data_utils/datasets/basedataset.py
deleted file mode 100644
index 084d6b4e..00000000
--- a/OpenCOOD/opencood/data_utils/datasets/basedataset.py
+++ /dev/null
@@ -1,619 +0,0 @@
-"""
-Basedataset class for all kinds of fusion.
-"""
-
-from __future__ import annotations
-
-import os
-import math
-import logging
-from collections import OrderedDict
-from typing import TYPE_CHECKING
-
-import torch
-import numpy as np
-from torch.utils.data import Dataset
-
-import opencood.utils.pcd_utils as pcd_utils
-from opencood.data_utils.augmentor.data_augmentor import DataAugmentor
-from opencood.hypes_yaml.yaml_utils import load_yaml
-from opencood.utils.pcd_utils import downsample_lidar_minimum
-from opencood.utils.transformation_utils import x1_to_x2
-
-if TYPE_CHECKING:
- from opencood.models.communication_adapters.base import ModelCommunicationAdapter
-
-logger = logging.getLogger("cavise.opencda.OpenCOOD.opencood.data_utils.datasets.basedataset")
-
-
-class BaseDataset(Dataset):
- """
- Base dataset for all kinds of fusion. Mainly used to initialize the
- database and associate the __get_item__ index with the correct timestamp
- and scenario.
-
- Parameters
- __________
- params : dict
- The dictionary contains all parameters for training/testing.
-
- visualize : false
- If set to true, the raw point cloud will be saved in the memory
- for visualization.
-
- Attributes
- ----------
- scenario_database : OrderedDict
- A structured dictionary contains all file information.
-
- len_record : list
- The list to record each scenario's data length. This is used to
- retrieve the correct index during training.
-
- pre_processor : opencood.pre_processor
- Used to preprocess the raw data.
-
- post_processor : opencood.post_processor
- Used to generate training labels and convert the model outputs to
- bbx formats.
-
- data_augmentor : opencood.data_augmentor
- Used to augment data.
-
- communication_adapter : ModelCommunicationAdapter | None
- Model-owned wire adapter attached by the CAPI model manager.
-
- """
-
- def __init__(self, params, visualize, train=True):
- self.params = params
- self.visualize = visualize
- self.train = train
- self.communication_adapter: ModelCommunicationAdapter | None = None
-
- self.pre_processor = None
- self.post_processor = None
- self.data_augmentor = DataAugmentor(params["data_augment"], train)
-
- # if the training/testing include noisy setting
- if "wild_setting" in params:
- self.seed = params["wild_setting"]["seed"]
- # whether to add time delay
- self.async_flag = params["wild_setting"]["async"]
- self.async_mode = "sim" if "async_mode" not in params["wild_setting"] else params["wild_setting"]["async_mode"]
- self.async_overhead = params["wild_setting"]["async_overhead"]
-
- # localization error
- self.loc_err_flag = params["wild_setting"]["loc_err"]
- self.xyz_noise_std = params["wild_setting"]["xyz_std"]
- self.ryp_noise_std = params["wild_setting"]["ryp_std"]
-
- # transmission data size
- self.data_size = params["wild_setting"]["data_size"] if "data_size" in params["wild_setting"] else 0
- self.transmission_speed = params["wild_setting"]["transmission_speed"] if "transmission_speed" in params["wild_setting"] else 27
- self.backbone_delay = params["wild_setting"]["backbone_delay"] if "backbone_delay" in params["wild_setting"] else 0
-
- else:
- self.async_flag = False
- self.async_overhead = 0 # ms
- self.async_mode = "sim"
- self.loc_err_flag = False
- self.xyz_noise_std = 0
- self.ryp_noise_std = 0
- self.data_size = 0 # Mb (Megabits)
- self.transmission_speed = 27 # Mbps
- self.backbone_delay = 0 # ms
-
- if "train_params" not in params or "max_cav" not in params["train_params"]:
- self.max_cav = 7
- else:
- self.max_cav = params["train_params"]["max_cav"]
-
- self.scenario_database = OrderedDict()
- self.len_record = []
-
- self.update_database()
-
- def update_database(self, memory_data=None):
- """
- Update the scenario database by re-scanning the root directory.
-
- This is an optimized method to refresh the database and length
- records without re-initializing the entire dataset object. It is
- particularly useful for real-time simulations where new data
- frames are added to the directory dynamically.
-
- Parameters
- ----------
- None
-
- Returns
- -------
- None
- """
- if memory_data is not None:
- self.scenario_database = memory_data
- self.len_record = [1]
- return
-
- if self.train:
- root_dir = self.params["root_dir"]
- else:
- root_dir = self.params["validate_dir"]
-
- if not os.path.exists(root_dir):
- logger.warning(f"Root directory {root_dir} does not exist.")
- return
-
- # first load all paths of different scenarios
- scenario_folders = sorted([os.path.join(root_dir, x) for x in os.listdir(root_dir) if os.path.isdir(os.path.join(root_dir, x))])
- # Structure: {scenario_id : {cav_1 : {timestamp1 : {yaml: path,
- # lidar: path, cameras:list of path}}}}
- self.scenario_database = OrderedDict()
- self.len_record = []
-
- # loop over all scenarios
- for i, scenario_folder in enumerate(scenario_folders):
- self.scenario_database.update({i: OrderedDict()})
-
- # at least 1 cav should show up
- cav_list = sorted([x for x in os.listdir(scenario_folder) if os.path.isdir(os.path.join(scenario_folder, x))])
- if len(cav_list) == 0:
- logger.warning(f"No CAVs found in {scenario_folder}. Skipping.")
- continue
-
- # roadside unit data's id is always negative, so here we want to
- # make sure they will be in the end of the list as they shouldn't
- # be ego vehicle.
- if "rsu" in cav_list[0]:
- cav_list = cav_list[1:] + [cav_list[0]]
-
- # loop over all CAV data
- for j, cav_id in enumerate(cav_list):
- if j > self.max_cav - 1:
- logger.warning(f"Too many CAVs and RSUs: {len(cav_list)}")
- logger.warning(f"Maximum is {self.max_cav}")
- break
- self.scenario_database[i][cav_id] = OrderedDict()
-
- # save all yaml files to the dictionary
- cav_path = os.path.join(scenario_folder, cav_id)
-
- # use the frame number as key, the full path as the values
- yaml_files = sorted([os.path.join(cav_path, x) for x in os.listdir(cav_path) if x.endswith(".yaml") and "additional" not in x])
- timestamps = self.extract_timestamps(yaml_files)
-
- for timestamp in timestamps:
- self.scenario_database[i][cav_id][timestamp] = OrderedDict()
-
- yaml_file = os.path.join(cav_path, timestamp + ".yaml")
- lidar_file = os.path.join(cav_path, timestamp + ".pcd")
- camera_files = self.load_camera_files(cav_path, timestamp)
-
- self.scenario_database[i][cav_id][timestamp]["yaml"] = yaml_file
- self.scenario_database[i][cav_id][timestamp]["lidar"] = lidar_file
- self.scenario_database[i][cav_id][timestamp]["camera0"] = camera_files
- # Assume all cavs will have the same timestamps length. Thus
- # we only need to calculate for the first vehicle in the
- # scene.
- if j == 0:
- # we regard the agent with the minimum id as the ego
- self.scenario_database[i][cav_id]["ego"] = True
- if not self.len_record:
- self.len_record.append(len(timestamps))
- else:
- prev_last = self.len_record[-1]
- self.len_record.append(prev_last + len(timestamps))
- else:
- self.scenario_database[i][cav_id]["ego"] = False
-
- if not self.len_record:
- self.len_record = [0]
-
- def __len__(self):
- return self.len_record[-1] if self.len_record else 0
-
- def __getitem__(self, idx):
- """
- Abstract method, needs to be define by the children class.
- """
- pass
-
- def retrieve_base_data(self, idx, cur_ego_pose_flag=True):
- """
- Given the index, return the corresponding data.
-
- Parameters
- ----------
- idx : int
- Index given by dataloader.
-
- cur_ego_pose_flag : bool
- Indicate whether to use current timestamp ego pose to calculate
- transformation matrix. If set to false, meaning when other cavs
- project their LiDAR point cloud to ego, they are projecting to
- past ego pose.
-
- Returns
- -------
- data : dict
- The dictionary contains loaded yaml params and lidar data for
- each cav.
- """
- # we loop the accumulated length list to see get the scenario index
- scenario_index = 0
- for i, ele in enumerate(self.len_record):
- if idx < ele:
- scenario_index = i
- break
- scenario_database = self.scenario_database[scenario_index]
-
- # check the timestamp index
- timestamp_index = idx if scenario_index == 0 else idx - self.len_record[scenario_index - 1]
- # retrieve the corresponding timestamp key
- timestamp_key = self.return_timestamp_key(scenario_database, timestamp_index)
- # calculate distance to ego for each cav
- ego_cav_content = self.calc_dist_to_ego(scenario_database, timestamp_key)
-
- data = OrderedDict()
- # load files for all CAVs
- for cav_id, cav_content in scenario_database.items():
- data[cav_id] = OrderedDict()
- data[cav_id]["ego"] = cav_content["ego"]
-
- # calculate delay for this vehicle
- timestamp_delay = self.time_delay_calculation(cav_content["ego"])
-
- if timestamp_index - timestamp_delay <= 0:
- timestamp_delay = timestamp_index
- timestamp_index_delay = max(0, timestamp_index - timestamp_delay)
- timestamp_key_delay = self.return_timestamp_key(scenario_database, timestamp_index_delay)
- # add time delay to vehicle parameters
- data[cav_id]["time_delay"] = timestamp_delay
- # load the corresponding data into the dictionary
- data[cav_id]["params"] = self.reform_param(cav_content, ego_cav_content, timestamp_key, timestamp_key_delay, cur_ego_pose_flag)
- data[cav_id]["lidar_np"] = (
- cav_content[timestamp_key_delay]["lidar_np"]
- if "lidar_np" in cav_content[timestamp_key_delay]
- else pcd_utils.pcd_to_np(cav_content[timestamp_key_delay]["lidar"])
- )
- if "spoofing_mask" in cav_content[timestamp_key_delay]:
- data[cav_id]["spoofing_mask"] = cav_content[timestamp_key_delay]["spoofing_mask"]
- return data
-
- @staticmethod
- def extract_timestamps(yaml_files):
- """
- Given the list of the yaml files, extract the mocked timestamps.
-
- Parameters
- ----------
- yaml_files : list
- The full path of all yaml files of ego vehicle
-
- Returns
- -------
- timestamps : list
- The list containing timestamps only.
- """
- timestamps = []
-
- for file in yaml_files:
- res = file.split("/")[-1]
-
- timestamp = res.replace(".yaml", "")
- timestamps.append(timestamp)
-
- return timestamps
-
- @staticmethod
- def return_timestamp_key(scenario_database, timestamp_index):
- """
- Given the timestamp index, return the correct timestamp key, e.g.
- 2 --> '000078'.
-
- Parameters
- ----------
- scenario_database : OrderedDict
- The dictionary contains all contents in the current scenario.
-
- timestamp_index : int
- The index for timestamp.
-
- Returns
- -------
- timestamp_key : str
- The timestamp key saved in the cav dictionary.
- """
- # get all timestamp keys
- timestamp_keys = list(scenario_database.items())[0][1]
- # retrieve the correct index
- timestamp_key = list(timestamp_keys.items())[timestamp_index][0]
-
- return timestamp_key
-
- def calc_dist_to_ego(self, scenario_database, timestamp_key):
- """
- Calculate the distance to ego for each cav.
- """
- ego_lidar_pose = None
- ego_cav_content = None
- # Find ego pose first
- for cav_id, cav_content in scenario_database.items():
- if cav_content["ego"]:
- ego_cav_content = cav_content
- ego_lidar_pose = (
- cav_content[timestamp_key]["params"] if "params" in cav_content[timestamp_key] else load_yaml(cav_content[timestamp_key]["yaml"])
- )["lidar_pose"]
- break
-
- assert ego_lidar_pose is not None
-
- # calculate the distance
- for cav_id, cav_content in scenario_database.items():
- cur_lidar_pose = (
- cav_content[timestamp_key]["params"] if "params" in cav_content[timestamp_key] else load_yaml(cav_content[timestamp_key]["yaml"])
- )["lidar_pose"]
-
- dx = cur_lidar_pose[0] - ego_lidar_pose[0]
- dy = cur_lidar_pose[1] - ego_lidar_pose[1]
- distance = math.hypot(dx, dy)
-
- cav_content["distance_to_ego"] = distance
- scenario_database.update({cav_id: cav_content})
-
- return ego_cav_content
-
- def time_delay_calculation(self, ego_flag):
- """
- Calculate the time delay for a certain vehicle.
-
- Parameters
- ----------
- ego_flag : boolean
- Whether the current cav is ego.
-
- Return
- ------
- time_delay : int
- The time delay quantization.
- """
- # there is not time delay for ego vehicle
- if ego_flag:
- return 0
- # time delay real mode
- if self.async_mode == "real":
- # in the real mode, time delay = systematic async time + data
- # transmission time + backbone computation time
- overhead_noise = np.random.uniform(0, self.async_overhead)
- tc = self.data_size / self.transmission_speed * 1000
- time_delay = int(overhead_noise + tc + self.backbone_delay)
- elif self.async_mode == "sim":
- # in the simulation mode, the time delay is constant
- time_delay = np.abs(self.async_overhead)
-
- # the data is 10 hz for both opv2v and v2x-set
- # todo: it may not be true for other dataset like DAIR-V2X and V2X-Sim
- time_delay = time_delay // 100
- return time_delay if self.async_flag else 0
-
- def add_loc_noise(self, pose, xyz_std, ryp_std):
- """
- Add localization noise to the pose.
-
- Parameters
- ----------
- pose : list
- x,y,z,roll,yaw,pitch
-
- xyz_std : float
- std of the gaussian noise on xyz
-
- ryp_std : float
- std of the gaussian noise
- """
- np.random.seed(self.seed)
- xyz_noise = np.random.normal(0, xyz_std, 3)
- ryp_std = np.random.normal(0, ryp_std, 3)
- noise_pose = [pose[0] + xyz_noise[0], pose[1] + xyz_noise[1], pose[2] + xyz_noise[2], pose[3], pose[4] + ryp_std[1], pose[5]]
- return noise_pose
-
- def reform_param(self, cav_content, ego_content, timestamp_cur, timestamp_delay, cur_ego_pose_flag):
- """
- Reform the data params with current timestamp object groundtruth and
- delay timestamp LiDAR pose for other CAVs.
-
- Parameters
- ----------
- cav_content : dict
- Dictionary that contains all file paths in the current cav/rsu.
-
- ego_content : dict
- Ego vehicle content.
-
- timestamp_cur : str
- The current timestamp.
-
- timestamp_delay : str
- The delayed timestamp.
-
- cur_ego_pose_flag : bool
- Whether use current ego pose to calculate transformation matrix.
-
- Return
- ------
- The merged parameters.
- """
- cur_params = (
- cav_content[timestamp_cur]["params"] if "params" in cav_content[timestamp_cur] else load_yaml(cav_content[timestamp_cur]["yaml"])
- ).copy()
- delay_params = (
- cav_content[timestamp_delay]["params"] if "params" in cav_content[timestamp_delay] else load_yaml(cav_content[timestamp_delay]["yaml"])
- ).copy()
-
- cur_ego_params = (
- ego_content[timestamp_cur]["params"] if "params" in ego_content[timestamp_cur] else load_yaml(ego_content[timestamp_cur]["yaml"])
- )
- delay_ego_params = (
- ego_content[timestamp_delay]["params"] if "params" in ego_content[timestamp_delay] else load_yaml(ego_content[timestamp_delay]["yaml"])
- )
-
- # we need to calculate the transformation matrix from cav to ego
- # at the delayed timestamp
- delay_cav_lidar_pose = delay_params["lidar_pose"]
- delay_ego_lidar_pose = delay_ego_params["lidar_pose"]
-
- cur_ego_lidar_pose = cur_ego_params["lidar_pose"]
- cur_cav_lidar_pose = cur_params["lidar_pose"]
-
- if not cav_content["ego"] and self.loc_err_flag:
- delay_cav_lidar_pose = self.add_loc_noise(delay_cav_lidar_pose, self.xyz_noise_std, self.ryp_noise_std)
- cur_cav_lidar_pose = self.add_loc_noise(cur_cav_lidar_pose, self.xyz_noise_std, self.ryp_noise_std)
-
- if cur_ego_pose_flag:
- transformation_matrix = x1_to_x2(delay_cav_lidar_pose, cur_ego_lidar_pose)
- spatial_correction_matrix = np.eye(4)
- else:
- transformation_matrix = x1_to_x2(delay_cav_lidar_pose, delay_ego_lidar_pose)
- spatial_correction_matrix = x1_to_x2(delay_ego_lidar_pose, cur_ego_lidar_pose)
- # This is only used for late fusion, as it did the transformation
- # in the postprocess, so we want the gt object transformation use
- # the correct one
- gt_transformation_matrix = x1_to_x2(cur_cav_lidar_pose, cur_ego_lidar_pose)
-
- # we always use current timestamp's gt bbx to gain a fair evaluation
- delay_params["vehicles"] = cur_params["vehicles"]
- delay_params["transformation_matrix"] = transformation_matrix
- delay_params["gt_transformation_matrix"] = gt_transformation_matrix
- delay_params["spatial_correction_matrix"] = spatial_correction_matrix
-
- return delay_params
-
- @staticmethod
- def load_camera_files(cav_path, timestamp):
- """
- Retrieve the paths to all camera files.
-
- Parameters
- ----------
- cav_path : str
- The full file path of current cav.
-
- timestamp : str
- Current timestamp
-
- Returns
- -------
- camera_files : list
- The list containing all camera png file paths.
- """
- camera0_file = os.path.join(cav_path, timestamp + "_camera0.png")
- camera1_file = os.path.join(cav_path, timestamp + "_camera1.png")
- camera2_file = os.path.join(cav_path, timestamp + "_camera2.png")
- camera3_file = os.path.join(cav_path, timestamp + "_camera3.png")
- return [camera0_file, camera1_file, camera2_file, camera3_file]
-
- def project_points_to_bev_map(self, points, ratio=0.1):
- """
- Project points to BEV occupancy map with default ratio=0.1.
-
- Parameters
- ----------
- points : np.ndarray
- (N, 3) / (N, 4)
-
- ratio : float
- Discretization parameters. Default is 0.1.
-
- Returns
- -------
- bev_map : np.ndarray
- BEV occupancy map including projected points
- with shape (img_row, img_col).
-
- """
- return self.pre_processor.project_points_to_bev_map(points, ratio)
-
- def augment(self, lidar_np, object_bbx_center, object_bbx_mask):
- """
- Given the raw point cloud, augment by flipping and rotation.
-
- Parameters
- ----------
- lidar_np : np.ndarray
- (n, 4) shape
-
- object_bbx_center : np.ndarray
- (n, 7) shape to represent bbx's x, y, z, h, w, l, yaw
-
- object_bbx_mask : np.ndarray
- Indicate which elements in object_bbx_center are padded.
- """
- tmp_dict = {"lidar_np": lidar_np, "object_bbx_center": object_bbx_center, "object_bbx_mask": object_bbx_mask}
- tmp_dict = self.data_augmentor.forward(tmp_dict)
-
- lidar_np = tmp_dict["lidar_np"]
- object_bbx_center = tmp_dict["object_bbx_center"]
- object_bbx_mask = tmp_dict["object_bbx_mask"]
-
- return lidar_np, object_bbx_center, object_bbx_mask
-
- def collate_batch_train(self, batch):
- """
- Customized collate function for pytorch dataloader during training
- for early and late fusion dataset.
-
- Parameters
- ----------
- batch : dict
-
- Returns
- -------
- batch : dict
- Reformatted batch.
- """
- # during training, we only care about ego.
- output_dict = {"ego": {}}
-
- object_bbx_center = []
- object_bbx_mask = []
- processed_lidar_list = []
- label_dict_list = []
-
- if self.visualize:
- origin_lidar = []
-
- for i in range(len(batch)):
- ego_dict = batch[i]["ego"]
- object_bbx_center.append(ego_dict["object_bbx_center"])
- object_bbx_mask.append(ego_dict["object_bbx_mask"])
- processed_lidar_list.append(ego_dict["processed_lidar"])
- label_dict_list.append(ego_dict["label_dict"])
-
- if self.visualize:
- origin_lidar.append(ego_dict["origin_lidar"])
-
- # convert to numpy, (B, max_num, 7)
- object_bbx_center = torch.from_numpy(np.array(object_bbx_center))
- object_bbx_mask = torch.from_numpy(np.array(object_bbx_mask))
-
- processed_lidar_torch_dict = self.pre_processor.collate_batch(processed_lidar_list)
- label_torch_dict = self.post_processor.collate_batch(label_dict_list)
- output_dict["ego"].update(
- {
- "object_bbx_center": object_bbx_center,
- "object_bbx_mask": object_bbx_mask,
- "processed_lidar": processed_lidar_torch_dict,
- "label_dict": label_torch_dict,
- }
- )
- if self.visualize:
- origin_lidar = np.array(downsample_lidar_minimum(pcd_np_list=origin_lidar))
- origin_lidar = torch.from_numpy(origin_lidar)
- output_dict["ego"].update({"origin_lidar": origin_lidar})
-
- return output_dict
-
- def visualize_result(self, pred_box_tensor, gt_tensor, pcd, show_vis, save_path, dataset=None):
- # visualize the model output
- self.post_processor.visualize(pred_box_tensor, gt_tensor, pcd, show_vis, save_path, dataset=dataset)
diff --git a/OpenCOOD/opencood/data_utils/datasets/early_fusion_dataset.py b/OpenCOOD/opencood/data_utils/datasets/early_fusion_dataset.py
deleted file mode 100644
index fbc408ca..00000000
--- a/OpenCOOD/opencood/data_utils/datasets/early_fusion_dataset.py
+++ /dev/null
@@ -1,451 +0,0 @@
-"""
-Dataset class for early fusion
-"""
-
-import math
-import logging
-from collections import OrderedDict
-from collections.abc import Callable
-from typing import Any
-
-import numpy as np
-import numpy.typing as npt
-import torch
-
-import opencood.data_utils.datasets
-from opencood.utils import box_utils
-from opencood.data_utils.post_processor import build_postprocessor
-from opencood.data_utils.datasets import basedataset
-from opencood.models.communication_adapters import PoseFrameMetadata
-from opencood.data_utils.pre_processor import build_preprocessor
-from opencood.utils.pcd_utils import downsample_lidar_minimum
-from opencood.utils.transformation_utils import x1_to_x2
-
-logger = logging.getLogger("cavise.opencda.OpenCOOD.opencood.data_utils.datasets.early_fusion_dataset")
-
-
-class EarlyFusionDataset(basedataset.BaseDataset):
- """
- This dataset is used for early fusion, where each CAV transmit the raw
- point cloud to the ego vehicle.
- """
-
- def __init__(self, params, visualize, train=True, payload_handler=None):
- super(EarlyFusionDataset, self).__init__(params, visualize, train)
- self.pre_processor = build_preprocessor(params["preprocess"], train)
- self.post_processor = build_postprocessor(params["postprocess"], train)
-
- self.payload_handler = payload_handler
- self.module_name = "OpenCOOD.EarlyFusionDataset"
-
- def __find_ego_vehicle(self, base_data_dict):
- ego_id = -1
- ego_lidar_pose = []
-
- # first find the ego vehicle's lidar pose
- for cav_id, cav_content in base_data_dict.items():
- if cav_content["ego"]:
- ego_id = cav_id
- ego_lidar_pose = cav_content["params"]["lidar_pose"]
- break
-
- assert ego_id != -1
- assert len(ego_lidar_pose) > 0
-
- return ego_id, ego_lidar_pose
-
- def extract_data(
- self,
- idx: int,
- agent_payload_builder: Callable[[npt.NDArray[Any], PoseFrameMetadata], object],
- ) -> None:
- """
- Publish sender-local observations through the communication adapter.
-
- Parameters
- ----------
- idx : int
- Dataset frame to publish.
- agent_payload_builder : Callable
- Adapter callback converting one local observation into its wire
- representation.
- """
- base_data_dict = self.retrieve_base_data(idx)
-
- if self.payload_handler is not None:
- for cav_id, selected_cav_base in base_data_dict.items():
- lidar_points, _ = self.__prepare_local_lidar(selected_cav_base)
- payload = agent_payload_builder(
- lidar_points,
- PoseFrameMetadata(
- lidar_pose=selected_cav_base["params"]["lidar_pose"],
- capture_frame=idx - int(selected_cav_base["time_delay"]),
- ),
- )
- self.payload_handler.set_opencda_payload(cav_id, self.module_name, payload)
-
- def __process_with_messages(self, ego_id, ego_lidar_pose, base_data_dict):
- projected_lidar_stack = []
-
- ego_cav_base = base_data_dict.get(ego_id)
- ego_cav_processed = self.get_item_single_car(ego_cav_base, ego_lidar_pose)
-
- projected_lidar_stack.append(ego_cav_processed["projected_lidar"])
-
- if ego_id in self.payload_handler.current_artery_payload:
- for cav_id, _ in base_data_dict.items():
- if cav_id == ego_id:
- continue
- raw_payload = self.payload_handler.get_artery_payload(ego_id, cav_id, self.module_name)
- if raw_payload is None:
- continue
- if self.communication_adapter is None:
- raise RuntimeError("Early-fusion payload decoding requires a communication adapter")
- decoded_payload = self.communication_adapter.decode_received_payload(
- raw_payload,
- ego_lidar_pose,
- )
- projected_lidar_stack.append(decoded_payload["projected_lidar"])
-
- return {"projected_lidar_stack": projected_lidar_stack}
-
- def __process_without_messages(self, ego_lidar_pose, base_data_dict):
- projected_lidar_stack = []
-
- # loop over all CAVs to process information
- for cav_id, selected_cav_base in base_data_dict.items():
- # check if the cav is within the communication range with ego
- dx = selected_cav_base["params"]["lidar_pose"][0] - ego_lidar_pose[0]
- dy = selected_cav_base["params"]["lidar_pose"][1] - ego_lidar_pose[1]
- distance = math.hypot(dx, dy)
-
- if distance > opencood.data_utils.datasets.COM_RANGE:
- continue
-
- selected_cav_processed = self.get_item_single_car(selected_cav_base, ego_lidar_pose)
- projected_lidar_stack.append(selected_cav_processed["projected_lidar"])
-
- return {"projected_lidar_stack": projected_lidar_stack}
-
- def build_local_supervision(self, base_data_dict: dict[str, Any], ego_lidar_pose: list[float]) -> dict[str, Any]:
- """
- Build ego-frame ground truth from the local scene snapshot.
-
- Parameters
- ----------
- base_data_dict : dict[str, Any]
- Complete local dataset snapshot for the current scene.
- ego_lidar_pose : list[float]
- Current ego LiDAR pose in world coordinates.
-
- Returns
- -------
- dict[str, Any]
- Ground-truth boxes, mask, and object identifiers.
- """
- object_bbx_center, object_bbx_mask, object_ids = self.post_processor.generate_object_center(list(base_data_dict.values()), ego_lidar_pose)
- return {
- "object_bbx_center": object_bbx_center,
- "object_bbx_mask": object_bbx_mask,
- "object_ids": object_ids,
- }
-
- def build_visualization_context(
- self,
- ego_id: str,
- ego_lidar_pose: list[float],
- base_data_dict: dict[str, Any],
- ) -> dict[str, Any]:
- """
- Build point-cloud visualization data from the local scene.
-
- Parameters
- ----------
- ego_id : str
- Identifier of the receiving ego agent.
- ego_lidar_pose : list[float]
- Current ego LiDAR pose in world coordinates.
- base_data_dict : dict[str, Any]
- Local dataset snapshot, independent of delivered messages.
-
- Returns
- -------
- dict[str, Any]
- Per-agent point clouds, identities, roles, and spoofing masks.
- """
- if not self.visualize:
- return {}
-
- projected_lidar_stack = []
- projected_lidar_roles = []
- projected_lidar_agent_ids = []
- projected_lidar_spoofing_masks = []
- lidar_range = self.params["preprocess"]["cav_lidar_range"]
-
- for cav_id, cav_base in base_data_dict.items():
- cav_processed = self.get_item_single_car(cav_base, ego_lidar_pose)
- projected_lidar = cav_processed["projected_lidar"]
- range_mask = (
- (projected_lidar[:, 0] > lidar_range[0])
- & (projected_lidar[:, 0] < lidar_range[3])
- & (projected_lidar[:, 1] > lidar_range[1])
- & (projected_lidar[:, 1] < lidar_range[4])
- & (projected_lidar[:, 2] > lidar_range[2])
- & (projected_lidar[:, 2] < lidar_range[5])
- )
- projected_lidar_stack.append(projected_lidar[range_mask])
- projected_lidar_roles.append("ego" if cav_id == ego_id else "other")
- projected_lidar_agent_ids.append(cav_id)
- projected_lidar_spoofing_masks.append(cav_processed["projected_lidar_spoofing_mask"][range_mask])
-
- return {
- "origin_lidar": np.vstack(projected_lidar_stack),
- "origin_lidar_by_agent": projected_lidar_stack,
- "origin_lidar_roles": projected_lidar_roles,
- "origin_lidar_agent_ids": projected_lidar_agent_ids,
- "origin_lidar_spoofing_masks": projected_lidar_spoofing_masks,
- }
-
- def assemble_inference_sample(
- self,
- inference_input: dict[str, Any],
- local_supervision: dict[str, Any],
- visualization_context: dict[str, Any],
- ) -> OrderedDict[str, dict[str, Any]]:
- """
- Combine delivered LiDAR with local supervision and visualization.
-
- Parameters
- ----------
- inference_input : dict[str, Any]
- Ego LiDAR plus point clouds from messages that were delivered.
- local_supervision : dict[str, Any]
- Ground truth produced solely from the local dataset snapshot.
- visualization_context : dict[str, Any]
- Visualization data produced solely from the local snapshot.
-
- Returns
- -------
- collections.OrderedDict
- Early-fusion sample ready for collation.
- """
- projected_lidar_stack = np.vstack(inference_input["projected_lidar_stack"])
- object_bbx_center = local_supervision["object_bbx_center"]
- object_bbx_mask = local_supervision["object_bbx_mask"]
-
- projected_lidar_stack, object_bbx_center, object_bbx_mask = self.augment(
- projected_lidar_stack,
- object_bbx_center,
- object_bbx_mask,
- )
-
- lidar_range = self.params["preprocess"]["cav_lidar_range"]
- lidar_mask = (
- (projected_lidar_stack[:, 0] > lidar_range[0])
- & (projected_lidar_stack[:, 0] < lidar_range[3])
- & (projected_lidar_stack[:, 1] > lidar_range[1])
- & (projected_lidar_stack[:, 1] < lidar_range[4])
- & (projected_lidar_stack[:, 2] > lidar_range[2])
- & (projected_lidar_stack[:, 2] < lidar_range[5])
- )
- projected_lidar_stack = projected_lidar_stack[lidar_mask]
-
- valid_indices = np.flatnonzero(object_bbx_mask == 1)
- valid_object_ids = [local_supervision["object_ids"][index] for index in valid_indices]
- valid_boxes, range_mask = box_utils.mask_boxes_outside_range_numpy(
- object_bbx_center[valid_indices],
- lidar_range,
- self.params["postprocess"]["order"],
- return_mask=True,
- )
- object_bbx_center.fill(0)
- object_bbx_mask.fill(0)
- object_bbx_center[: valid_boxes.shape[0]] = valid_boxes
- object_bbx_mask[: valid_boxes.shape[0]] = 1
-
- anchor_box = self.post_processor.generate_anchor_box()
- label_dict = self.post_processor.generate_label(
- gt_box_center=object_bbx_center,
- anchors=anchor_box,
- mask=object_bbx_mask,
- )
- ego_sample = {
- "object_bbx_center": object_bbx_center,
- "object_bbx_mask": object_bbx_mask,
- "object_ids": [object_id for object_id, keep in zip(valid_object_ids, range_mask) if keep],
- "anchor_box": anchor_box,
- "processed_lidar": self.pre_processor.preprocess(projected_lidar_stack),
- "label_dict": label_dict,
- **visualization_context,
- }
- return OrderedDict({"ego": ego_sample})
-
- def __getitem__(self, idx):
- base_data_dict = self.retrieve_base_data(idx)
- ego_id, ego_lidar_pose = self.__find_ego_vehicle(base_data_dict)
-
- if self.payload_handler is not None:
- inference_input = self.__process_with_messages(ego_id, ego_lidar_pose, base_data_dict)
- else:
- inference_input = self.__process_without_messages(ego_lidar_pose, base_data_dict)
-
- local_supervision = self.build_local_supervision(base_data_dict, ego_lidar_pose)
- visualization_context = self.build_visualization_context(ego_id, ego_lidar_pose, base_data_dict)
- return self.assemble_inference_sample(inference_input, local_supervision, visualization_context)
-
- def get_item_single_car(self, selected_cav_base, ego_pose):
- """
- Project one agent's LiDAR into ego coordinates.
-
- Parameters
- ----------
- selected_cav_base : dict
- The dictionary contains a single CAV's raw information.
- ego_pose : list
- The ego vehicle lidar pose under world coordinate.
-
- Returns
- -------
- selected_cav_processed : dict
- The dictionary contains the cav's processed information.
- """
- # calculate the transformation matrix
- transformation_matrix = x1_to_x2(selected_cav_base["params"]["lidar_pose"], ego_pose)
-
- lidar_np, spoofing_mask = self.__prepare_local_lidar(selected_cav_base)
- # project the lidar to ego space
- lidar_np[:, :3] = box_utils.project_points_by_matrix_torch(lidar_np[:, :3], transformation_matrix)
-
- return {
- "projected_lidar": lidar_np,
- "projected_lidar_spoofing_mask": spoofing_mask,
- }
-
- @staticmethod
- def __prepare_local_lidar(
- selected_cav_base: dict[str, Any],
- ) -> tuple[np.ndarray, np.ndarray]:
- """
- Shuffle and remove ego returns without changing coordinate frames.
-
- Parameters
- ----------
- selected_cav_base : dict[str, Any]
- Raw dataset entry for one sending agent.
-
- Returns
- -------
- tuple[numpy.ndarray, numpy.ndarray]
- Sender-local LiDAR points and their aligned spoofing mask.
- """
- lidar_np = selected_cav_base["lidar_np"]
- spoofing_mask = np.asarray(
- selected_cav_base.get(
- "spoofing_mask",
- np.zeros((lidar_np.shape[0],), dtype=np.bool_),
- ),
- dtype=np.bool_,
- )
-
- shuffle_idx = np.random.permutation(lidar_np.shape[0])
- lidar_np = np.array(lidar_np[shuffle_idx], copy=True)
- spoofing_mask = np.array(spoofing_mask[shuffle_idx], copy=True)
-
- ego_mask = (lidar_np[:, 0] >= -1.95) & (lidar_np[:, 0] <= 2.95) & (lidar_np[:, 1] >= -1.1) & (lidar_np[:, 1] <= 1.1)
- keep_mask = np.logical_not(ego_mask)
- return lidar_np[keep_mask], spoofing_mask[keep_mask]
-
- def collate_batch_test(self, batch):
- """
- Customized collate function for pytorch dataloader during testing
- for late fusion dataset.
-
- Parameters
- ----------
- batch : dict
-
- Returns
- -------
- batch : dict
- Reformatted batch.
- """
- # currently, we only support batch size of 1 during testing
- assert len(batch) <= 1, "Batch size 1 is required during testing!"
- batch = batch[0]
-
- output_dict = {}
-
- for cav_id, cav_content in batch.items():
- output_dict.update({cav_id: {}})
- # shape: (1, max_num, 7)
- object_bbx_center = torch.from_numpy(np.array([cav_content["object_bbx_center"]]))
- object_bbx_mask = torch.from_numpy(np.array([cav_content["object_bbx_mask"]]))
- object_ids = cav_content["object_ids"]
-
- # the anchor box is the same for all bounding boxes usually, thus
- # we don't need the batch dimension.
- if cav_content["anchor_box"] is not None:
- output_dict[cav_id].update({"anchor_box": torch.from_numpy(np.array(cav_content["anchor_box"]))})
- if self.visualize:
- origin_lidar = [cav_content["origin_lidar"]]
-
- # processed lidar dictionary
- processed_lidar_torch_dict = self.pre_processor.collate_batch([cav_content["processed_lidar"]])
- # label dictionary
- label_torch_dict = self.post_processor.collate_batch([cav_content["label_dict"]])
-
- # save the transformation matrix (4, 4) to ego vehicle
- transformation_matrix_torch = torch.from_numpy(np.identity(4)).float()
-
- output_dict[cav_id].update(
- {
- "object_bbx_center": object_bbx_center,
- "object_bbx_mask": object_bbx_mask,
- "processed_lidar": processed_lidar_torch_dict,
- "label_dict": label_torch_dict,
- "object_ids": object_ids,
- "transformation_matrix": transformation_matrix_torch,
- }
- )
-
- if self.visualize:
- origin_lidar = np.array(downsample_lidar_minimum(pcd_np_list=origin_lidar))
- origin_lidar = torch.from_numpy(origin_lidar)
- output_dict[cav_id].update({"origin_lidar": origin_lidar})
- if "origin_lidar_by_agent" in cav_content:
- output_dict[cav_id].update(
- {
- "origin_lidar_by_agent": [torch.from_numpy(np.array(points)) for points in cav_content["origin_lidar_by_agent"]],
- "origin_lidar_roles": list(cav_content["origin_lidar_roles"]),
- "origin_lidar_agent_ids": list(cav_content["origin_lidar_agent_ids"]),
- "origin_lidar_spoofing_masks": [
- torch.from_numpy(np.asarray(mask, dtype=np.bool_)) for mask in cav_content["origin_lidar_spoofing_masks"]
- ],
- }
- )
-
- return output_dict
-
- def post_process(self, data_dict, output_dict):
- """
- Process the outputs of the model to 2D/3D bounding box.
-
- Parameters
- ----------
- data_dict : dict
- The dictionary containing the origin input data of model.
-
- output_dict :dict
- The dictionary containing the output of the model.
-
- Returns
- -------
- pred_box_tensor : torch.Tensor
- The tensor of prediction bounding box after NMS.
- gt_box_tensor : torch.Tensor
- The tensor of gt bounding box.
- """
- pred_box_tensor, pred_score = self.post_processor.post_process(data_dict, output_dict)
- gt_box_tensor = self.post_processor.generate_gt_bbx(data_dict)
-
- return pred_box_tensor, pred_score, gt_box_tensor
diff --git a/OpenCOOD/opencood/data_utils/datasets/early_fusion_vis_dataset.py b/OpenCOOD/opencood/data_utils/datasets/early_fusion_vis_dataset.py
deleted file mode 100644
index 65f73673..00000000
--- a/OpenCOOD/opencood/data_utils/datasets/early_fusion_vis_dataset.py
+++ /dev/null
@@ -1,170 +0,0 @@
-"""
-This is a dataset for early fusion visualization only.
-"""
-
-from collections import OrderedDict
-
-import numpy as np
-import torch
-
-from opencood.utils import box_utils
-from opencood.data_utils.post_processor import build_postprocessor
-from opencood.data_utils.datasets import basedataset
-from opencood.data_utils.pre_processor import build_preprocessor
-from opencood.utils.pcd_utils import mask_points_by_range, mask_ego_points, shuffle_points, downsample_lidar_minimum
-from opencood.utils.transformation_utils import x1_to_x2
-
-
-# TODO: Вообше не понятно зачем он нужен, вроде нигде не используется
-class EarlyFusionVisDataset(basedataset.BaseDataset):
- def __init__(self, params, visualize, train=True):
- super(EarlyFusionVisDataset, self).__init__(params, visualize, train)
- self.pre_processor = build_preprocessor(params["preprocess"], train)
- self.post_processor = build_postprocessor(params["postprocess"], train)
-
- def __getitem__(self, idx):
- base_data_dict = self.retrieve_base_data(idx)
-
- processed_data_dict = OrderedDict()
- processed_data_dict["ego"] = {}
-
- ego_id = -1
- ego_lidar_pose = []
-
- # first find the ego vehicle's lidar pose
- for cav_id, cav_content in base_data_dict.items():
- if cav_content["ego"]:
- ego_id = cav_id
- ego_lidar_pose = cav_content["params"]["lidar_pose"]
- break
-
- assert ego_id != -1
- assert len(ego_lidar_pose) > 0
-
- projected_lidar_stack = []
- object_stack = []
- object_id_stack = []
-
- # loop over all CAVs to process information
- for cav_id, selected_cav_base in base_data_dict.items():
- selected_cav_processed = self.get_item_single_car(selected_cav_base, ego_lidar_pose)
- # all these lidar and object coordinates are projected to ego
- # already.
- projected_lidar_stack.append(selected_cav_processed["projected_lidar"])
- object_stack.append(selected_cav_processed["object_bbx_center"])
- object_id_stack += selected_cav_processed["object_ids"]
-
- # exclude all repetitive objects
- unique_indices = [object_id_stack.index(x) for x in set(object_id_stack)]
- object_stack = np.vstack(object_stack)
- object_stack = object_stack[unique_indices]
-
- # make sure bounding boxes across all frames have the same number
- object_bbx_center = np.zeros((self.params["postprocess"]["max_num"], 7))
- mask = np.zeros(self.params["postprocess"]["max_num"])
- object_bbx_center[: object_stack.shape[0], :] = object_stack
- mask[: object_stack.shape[0]] = 1
-
- # convert list to numpy array, (N, 4)
- projected_lidar_stack = np.vstack(projected_lidar_stack)
-
- # data augmentation
- projected_lidar_stack, object_bbx_center, mask = self.augment(projected_lidar_stack, object_bbx_center, mask)
-
- # we do lidar filtering in the stacked lidar
- projected_lidar_stack = mask_points_by_range(projected_lidar_stack, self.params["preprocess"]["cav_lidar_range"])
- # augmentation may remove some of the bbx out of range
- object_bbx_center_valid = object_bbx_center[mask == 1]
- object_bbx_center_valid = box_utils.mask_boxes_outside_range_numpy(
- object_bbx_center_valid, self.params["preprocess"]["cav_lidar_range"], self.params["postprocess"]["order"]
- )
- mask[object_bbx_center_valid.shape[0] :] = 0
- object_bbx_center[: object_bbx_center_valid.shape[0]] = object_bbx_center_valid
- object_bbx_center[object_bbx_center_valid.shape[0] :] = 0
-
- processed_data_dict["ego"].update(
- {
- "object_bbx_center": object_bbx_center,
- "object_bbx_mask": mask,
- "object_ids": [object_id_stack[i] for i in unique_indices],
- "origin_lidar": projected_lidar_stack,
- }
- )
-
- return processed_data_dict
-
- def get_item_single_car(self, selected_cav_base, ego_pose):
- """
- Project the lidar and bbx to ego space first, and then do clipping.
-
- Parameters
- ----------
- selected_cav_base : dict
- The dictionary contains a single CAV's raw information.
- ego_pose : list
- The ego vehicle lidar pose under world coordinate.
-
- Returns
- -------
- selected_cav_processed : dict
- The dictionary contains the cav's processed information.
- """
- selected_cav_processed = {}
-
- # calculate the transformation matrix
- transformation_matrix = x1_to_x2(selected_cav_base["params"]["lidar_pose"], ego_pose)
-
- # retrieve objects under ego coordinates
- object_bbx_center, object_bbx_mask, object_ids = self.post_processor.generate_object_center([selected_cav_base], ego_pose)
-
- # filter lidar
- lidar_np = selected_cav_base["lidar_np"]
- lidar_np = shuffle_points(lidar_np)
- # remove points that hit itself
- lidar_np = mask_ego_points(lidar_np)
- # project the lidar to ego space
- lidar_np[:, :3] = box_utils.project_points_by_matrix_torch(lidar_np[:, :3], transformation_matrix)
-
- selected_cav_processed.update(
- {"object_bbx_center": object_bbx_center[object_bbx_mask == 1], "object_ids": object_ids, "projected_lidar": lidar_np}
- )
-
- return selected_cav_processed
-
- def collate_batch_train(self, batch):
- """
- Customized collate function for pytorch dataloader during training
- for late fusion dataset.
-
- Parameters
- ----------
- batch : dict
-
- Returns
- -------
- batch : dict
- Reformatted batch.
- """
- # during training, we only care about ego.
- output_dict = {"ego": {}}
-
- object_bbx_center = []
- object_bbx_mask = []
- origin_lidar = []
-
- for i in range(len(batch)):
- ego_dict = batch[i]["ego"]
- object_bbx_center.append(ego_dict["object_bbx_center"])
- object_bbx_mask.append(ego_dict["object_bbx_mask"])
- origin_lidar.append(ego_dict["origin_lidar"])
-
- # convert to numpy, (B, max_num, 7)
- object_bbx_center = torch.from_numpy(np.array(object_bbx_center))
- object_bbx_mask = torch.from_numpy(np.array(object_bbx_mask))
- output_dict["ego"].update({"object_bbx_center": object_bbx_center, "object_bbx_mask": object_bbx_mask})
-
- origin_lidar = np.array(downsample_lidar_minimum(pcd_np_list=origin_lidar))
- origin_lidar = torch.from_numpy(origin_lidar)
- output_dict["ego"].update({"origin_lidar": origin_lidar})
-
- return output_dict
diff --git a/OpenCOOD/opencood/data_utils/datasets/intermediate_fusion_dataset.py b/OpenCOOD/opencood/data_utils/datasets/intermediate_fusion_dataset.py
deleted file mode 100644
index 392e1072..00000000
--- a/OpenCOOD/opencood/data_utils/datasets/intermediate_fusion_dataset.py
+++ /dev/null
@@ -1,763 +0,0 @@
-"""
-Dataset class for intermediate fusion
-"""
-
-import math
-import logging
-from collections import OrderedDict
-from collections.abc import Callable
-from typing import Any
-
-import numpy as np
-import torch
-
-import opencood.data_utils.datasets
-import opencood.data_utils.post_processor as post_processor
-from opencood.utils import box_utils
-from opencood.data_utils.datasets import basedataset
-from opencood.models.communication_adapters import (
- InferenceInput,
- IntermediateMetadata,
- PoseFrameMetadata,
- V2XViTMetadata,
- build_inference_input,
- merge_inference_inputs,
-)
-from opencood.data_utils.pre_processor import build_preprocessor
-from opencood.utils.pcd_utils import pcd_to_np, mask_points_by_range, mask_ego_points, shuffle_points, downsample_lidar_minimum
-from opencood.utils.transformation_utils import x1_to_x2
-
-logger = logging.getLogger("cavise.opencda.OpenCOOD.opencood.data_utils.datasets.intermediate_fusion_dataset")
-
-PAIRWISE_METADATA_MODELS = frozenset(
- {
- "point_pillar_coalign",
- "point_pillar_v2vnet",
- "point_pillar_where2comm",
- }
-)
-V2X_VIT_METADATA_MODELS = frozenset({"point_pillar_transformer"})
-
-
-class IntermediateFusionDataset(basedataset.BaseDataset):
- """
- This class is for intermediate fusion where each vehicle transmit the
- deep features to ego.
- """
-
- def __init__(self, params, visualize, train=True, payload_handler=None):
- super(IntermediateFusionDataset, self).__init__(params, visualize, train)
-
- # if project first, cav's lidar will first be projected to
- # the ego's coordinate frame. otherwise, the feature will be
- # projected instead.
- self.proj_first = True
- if "proj_first" in params["fusion"]["args"] and not params["fusion"]["args"]["proj_first"]:
- self.proj_first = False
-
- # whether there is a time delay between the time that cav project
- # lidar to ego and the ego receive the delivered feature
- if "cur_ego_pose_flag" in params["fusion"]["args"]:
- self.cur_ego_pose_flag = params["fusion"]["args"]["cur_ego_pose_flag"]
- else:
- self.cur_ego_pose_flag = True
-
- self.pre_processor = build_preprocessor(params["preprocess"], train)
- self.post_processor = post_processor.build_postprocessor(params["postprocess"], train)
-
- self.payload_handler = payload_handler
- self.module_name = "OpenCOOD.IntermediateFusionDataset"
- self.model_name = params["model"]["core_method"]
-
- def extract_data(
- self,
- idx: int,
- agent_payload_builder: Callable[
- [InferenceInput, IntermediateMetadata | None],
- object,
- ],
- ) -> None:
- """
- Publish individual agent inputs through the selected model adapter.
-
- Parameters
- ----------
- idx : int
- Dataset frame to publish.
- agent_payload_builder : Callable
- Model adapter callback that encodes the sender-local input at the
- configured model's communication boundary.
- """
- base_data_dict = self.retrieve_base_data(idx, cur_ego_pose_flag=self.cur_ego_pose_flag)
- _, ego_lidar_pose = self.__find_ego_vehicle(base_data_dict)
-
- if self.payload_handler is not None:
- for cav_id, selected_cav_base in base_data_dict.items():
- selected_cav_processed = self.get_item_single_car(selected_cav_base, ego_lidar_pose)
- payload = agent_payload_builder(
- selected_cav_processed["inference_input"],
- self.build_model_metadata(cav_id, selected_cav_base, idx),
- )
- self.payload_handler.set_opencda_payload(cav_id, self.module_name, payload)
-
- def build_model_metadata(
- self,
- cav_id: str,
- selected_cav_base: dict[str, Any],
- receive_frame: int,
- ) -> IntermediateMetadata | None:
- """
- Build only the metadata consumed by the configured model.
-
- Parameters
- ----------
- cav_id : str
- Identifier of the sending CAV or RSU.
- selected_cav_base : dict[str, Any]
- Local dataset entry for the sending agent.
- receive_frame : int
- Dataset frame in which the model input is received.
-
- Returns
- -------
- IntermediateMetadata | None
- Model-specific metadata, or ``None`` for models that consume only
- the preprocessed inference input.
- """
- capture_frame = receive_frame - int(selected_cav_base["time_delay"])
- if self.model_name in PAIRWISE_METADATA_MODELS:
- return PoseFrameMetadata(
- lidar_pose=selected_cav_base["params"]["lidar_pose"],
- capture_frame=capture_frame,
- )
- if self.model_name in V2X_VIT_METADATA_MODELS:
- return V2XViTMetadata(
- velocity=float(selected_cav_base["params"]["ego_speed"] / 30),
- agent_type=1 if "rsu" in cav_id else 0,
- capture_frame=capture_frame,
- spatial_correction_matrix=selected_cav_base["params"]["spatial_correction_matrix"],
- )
- return None
-
- def __find_ego_vehicle(self, base_data_dict):
- ego_id = -1
- ego_lidar_pose = []
-
- # first find the ego vehicle's lidar pose
- for cav_id, cav_content in base_data_dict.items():
- if cav_content["ego"]:
- ego_id = cav_id
- ego_lidar_pose = cav_content["params"]["lidar_pose"]
- break
-
- assert cav_id == list(base_data_dict.keys())[0], "The first element in the OrderedDict must be ego"
- assert ego_id != -1
- assert len(ego_lidar_pose) > 0
-
- return ego_id, ego_lidar_pose
-
- def __retrieve_visualization_base_data(self, idx):
- scenario_index = 0
- for i, ele in enumerate(self.len_record):
- if idx < ele:
- scenario_index = i
- break
- scenario_database = self.scenario_database[scenario_index]
-
- timestamp_index = idx if scenario_index == 0 else idx - self.len_record[scenario_index - 1]
- timestamp_key = self.return_timestamp_key(scenario_database, timestamp_index)
- ego_cav_content = self.calc_dist_to_ego(scenario_database, timestamp_key)
-
- data = OrderedDict()
- for cav_id, cav_content in scenario_database.items():
- cav_snapshot = cav_content[timestamp_key]
- data[cav_id] = OrderedDict()
- data[cav_id]["ego"] = cav_content["ego"]
- data[cav_id]["time_delay"] = 0
- data[cav_id]["params"] = self.reform_param(
- cav_content,
- ego_cav_content,
- timestamp_key,
- timestamp_key,
- cur_ego_pose_flag=True,
- )
- data[cav_id]["lidar_np"] = cav_snapshot["lidar_np"] if "lidar_np" in cav_snapshot else pcd_to_np(cav_snapshot["lidar"])
- if "spoofing_mask" in cav_snapshot:
- data[cav_id]["spoofing_mask"] = cav_snapshot["spoofing_mask"]
- return data
-
- def __pad_spatial_matrix(self, matrix_list):
- matrix_list = np.stack(matrix_list)
- padding_eye = np.tile(np.eye(4)[None], (self.max_cav - len(matrix_list), 1, 1))
- return np.concatenate([matrix_list, padding_eye], axis=0)
-
- def __pad_to_max(self, lst, pad_value):
- return lst + (self.max_cav - len(lst)) * [pad_value]
-
- def __build_pairwise_transformation(self, metadata: list[PoseFrameMetadata]) -> np.ndarray:
- """
- Build pairwise transformations for the received agents.
-
- Parameters
- ----------
- metadata : list[PoseFrameMetadata]
- Pose metadata ordered exactly like the received model inputs.
-
- Returns
- -------
- numpy.ndarray
- Pairwise transformation matrix padded to ``max_cav``.
- """
- pairwise_t_matrix = np.zeros((self.max_cav, self.max_cav, 4, 4))
- if self.proj_first:
- pairwise_t_matrix[:, :] = np.identity(4)
- return pairwise_t_matrix
-
- for source_index, source_metadata in enumerate(metadata):
- for target_index, target_metadata in enumerate(metadata):
- pairwise_t_matrix[source_index, target_index] = x1_to_x2(
- source_metadata.lidar_pose,
- target_metadata.lidar_pose,
- )
- return pairwise_t_matrix
-
- def __assemble_model_metadata(
- self,
- metadata: list[IntermediateMetadata | None],
- receive_frame: int,
- ) -> dict[str, Any]:
- """
- Derive model tensors from received metadata.
-
- Parameters
- ----------
- metadata : list[IntermediateMetadata | None]
- Metadata ordered exactly like the available inference inputs.
- receive_frame : int
- Current receiver dataset frame.
-
- Returns
- -------
- dict[str, Any]
- Model fields derived from metadata.
-
- Raises
- ------
- TypeError
- If the received metadata does not match the configured model.
- """
- if self.model_name in PAIRWISE_METADATA_MODELS:
- if not all(isinstance(agent_metadata, PoseFrameMetadata) for agent_metadata in metadata):
- raise TypeError(f"{self.model_name} requires PoseFrameMetadata for every agent")
- pose_metadata = [agent_metadata for agent_metadata in metadata if isinstance(agent_metadata, PoseFrameMetadata)]
- return {"pairwise_t_matrix": self.__build_pairwise_transformation(pose_metadata)}
-
- if self.model_name in V2X_VIT_METADATA_MODELS:
- if not all(isinstance(agent_metadata, V2XViTMetadata) for agent_metadata in metadata):
- raise TypeError(f"{self.model_name} requires V2XViTMetadata for every agent")
- v2x_metadata = [agent_metadata for agent_metadata in metadata if isinstance(agent_metadata, V2XViTMetadata)]
- return {
- "velocity": self.__pad_to_max([agent_metadata.velocity for agent_metadata in v2x_metadata], 0.0),
- "time_delay": self.__pad_to_max(
- [float(receive_frame - agent_metadata.capture_frame) for agent_metadata in v2x_metadata],
- 0.0,
- ),
- "infra": self.__pad_to_max([agent_metadata.agent_type for agent_metadata in v2x_metadata], 0.0),
- "spatial_correction_matrix": self.__pad_spatial_matrix([agent_metadata.spatial_correction_matrix for agent_metadata in v2x_metadata]),
- }
-
- if any(agent_metadata is not None for agent_metadata in metadata):
- raise TypeError(f"{self.model_name} does not consume wire metadata")
- return {}
-
- @staticmethod
- def __build_model_data():
- return {
- "inference_inputs": [],
- "metadata": [],
- }
-
- def __append_processed_model_data(self, data, cav_id, cav_base, cav_processed, receive_frame):
- data["inference_inputs"].append(cav_processed["inference_input"])
- data["metadata"].append(self.build_model_metadata(cav_id, cav_base, receive_frame))
-
- def __agent_in_communication_range(self, cav_base, ego_lidar_pose):
- dx = cav_base["params"]["lidar_pose"][0] - ego_lidar_pose[0]
- dy = cav_base["params"]["lidar_pose"][1] - ego_lidar_pose[1]
- return math.hypot(dx, dy) <= opencood.data_utils.datasets.COM_RANGE
-
- def __project_lidar_for_visualization(self, cav_base):
- transformation_matrix = cav_base["params"]["transformation_matrix"]
- lidar_np = np.array(cav_base["lidar_np"], copy=True)
- lidar_np = mask_ego_points(lidar_np)
- if self.proj_first:
- lidar_np[:, :3] = box_utils.project_points_by_matrix_torch(lidar_np[:, :3], transformation_matrix)
- return mask_points_by_range(lidar_np, self.params["preprocess"]["cav_lidar_range"])
-
- def build_visualization_context(self, ego_id: str, base_data_dict: dict[str, Any]) -> dict[str, Any]:
- """
- Build visualization data from the local scene snapshot.
-
- Parameters
- ----------
- ego_id : str
- Identifier of the receiving ego agent.
- base_data_dict : dict[str, Any]
- Local dataset snapshot, independent of delivered messages.
-
- Returns
- -------
- dict[str, Any]
- Per-agent projected point clouds and their local identities.
- """
- if not self.visualize:
- return {
- "projected_lidar_stack": [],
- "projected_lidar_roles": [],
- "projected_lidar_agent_ids": [],
- }
-
- projected_lidar_stack = []
- projected_lidar_roles = []
- projected_lidar_agent_ids = []
-
- for cav_id in base_data_dict:
- projected_lidar_stack.append(self.__project_lidar_for_visualization(base_data_dict[cav_id]))
- projected_lidar_roles.append("ego" if cav_id == ego_id else "other")
- projected_lidar_agent_ids.append(cav_id)
-
- return {
- "projected_lidar_stack": projected_lidar_stack,
- "projected_lidar_roles": projected_lidar_roles,
- "projected_lidar_agent_ids": projected_lidar_agent_ids,
- }
-
- def __process_with_messages(self, ego_id, ego_lidar_pose, base_data_dict, receive_frame):
- return self.__process_encoded_messages(
- ego_id,
- ego_lidar_pose,
- base_data_dict,
- receive_frame,
- )
-
- def __process_encoded_messages(
- self,
- ego_id: str,
- ego_lidar_pose: list[float],
- base_data_dict: dict[str, Any],
- receive_frame: int,
- ) -> dict[str, Any]:
- """
- Assemble learned local and delivered features for model-side fusion.
-
- Parameters
- ----------
- ego_id : str
- Identifier of the receiving agent.
- ego_lidar_pose : list[float]
- Current receiver LiDAR pose.
- base_data_dict : dict[str, Any]
- Local scene observations.
- receive_frame : int
- Current dataset frame.
-
- Returns
- -------
- dict[str, Any]
- Successfully available learned features and their metadata.
- """
- if self.communication_adapter is None:
- raise RuntimeError("Intermediate feature decoding requires a communication adapter")
-
- ego_cav_base = base_data_dict[ego_id]
- ego_cav_processed = self.get_item_single_car(ego_cav_base, ego_lidar_pose)
- features = [
- self.communication_adapter.encode_local_intermediate_input(
- self,
- ego_cav_processed["inference_input"],
- )
- ]
- metadata = [self.build_model_metadata(ego_id, ego_cav_base, receive_frame)]
-
- if ego_id in self.payload_handler.current_artery_payload:
- for cav_id in base_data_dict:
- if cav_id == ego_id:
- continue
- raw_payload = self.payload_handler.get_artery_payload(
- ego_id,
- cav_id,
- self.module_name,
- )
- if raw_payload is None:
- continue
- decoded = self.communication_adapter.decode_received_payload(
- raw_payload,
- ego_lidar_pose,
- )
- metadata.append(decoded.pop("metadata"))
- features.append(decoded)
-
- return {
- "intermediate_features": features,
- "metadata": metadata,
- }
-
- def __process_without_messages(self, ego_id, ego_lidar_pose, base_data_dict, receive_frame):
- data = self.__build_model_data()
- for cav_id, selected_cav_base in base_data_dict.items():
- if not self.__agent_in_communication_range(selected_cav_base, ego_lidar_pose):
- continue
-
- selected_cav_processed = self.get_item_single_car(selected_cav_base, ego_lidar_pose)
- self.__append_processed_model_data(data, cav_id, selected_cav_base, selected_cav_processed, receive_frame)
-
- return data
-
- def build_local_supervision(self, base_data_dict: dict[str, Any], ego_lidar_pose: list[float]) -> dict[str, Any]:
- """
- Build ground truth and training targets from the local scene.
-
- Parameters
- ----------
- base_data_dict : dict[str, Any]
- Complete local dataset snapshot for the current scene.
- ego_lidar_pose : list[float]
- Current ego LiDAR pose in world coordinates.
-
- Returns
- -------
- dict[str, Any]
- Ground-truth boxes, object identifiers, anchors, and labels.
- """
- object_bbx_center, object_bbx_mask, object_ids = self.post_processor.generate_object_center(list(base_data_dict.values()), ego_lidar_pose)
- anchor_box = self.post_processor.generate_anchor_box()
- label_dict = self.post_processor.generate_label(
- gt_box_center=object_bbx_center,
- anchors=anchor_box,
- mask=object_bbx_mask,
- )
- return {
- "object_bbx_center": object_bbx_center,
- "object_bbx_mask": object_bbx_mask,
- "object_ids": object_ids,
- "anchor_box": anchor_box,
- "label_dict": label_dict,
- }
-
- def assemble_inference_sample(
- self,
- inference_input: dict[str, Any],
- local_supervision: dict[str, Any],
- visualization_context: dict[str, Any],
- receive_frame: int,
- ) -> OrderedDict[str, dict[str, Any]]:
- """
- Assemble model input with local supervision and visualization.
-
- Parameters
- ----------
- inference_input : dict[str, Any]
- Ego features plus features from messages that were delivered.
- local_supervision : dict[str, Any]
- Ground truth produced solely from the local dataset snapshot.
- visualization_context : dict[str, Any]
- Visualization data produced solely from the local snapshot.
- receive_frame : int
- Current receiver dataset frame.
-
- Returns
- -------
- collections.OrderedDict
- Dataset sample consumed by the collate and inference pipeline.
-
- Raises
- ------
- ValueError
- If model inputs and metadata entries are not aligned.
- """
- metadata: list[IntermediateMetadata | None] = inference_input["metadata"]
- if "intermediate_features" in inference_input:
- intermediate_features = inference_input["intermediate_features"]
- if len(metadata) != len(intermediate_features):
- raise ValueError("Every intermediate feature must have a matching metadata entry")
- feature_fields = set(intermediate_features[0])
- if any(set(features) != feature_fields for features in intermediate_features[1:]):
- raise TypeError("All agents must provide the same intermediate feature fields")
- merged_feature_dict = {
- field: (
- np.concatenate([features[field] for features in intermediate_features], axis=0)
- if isinstance(intermediate_features[0][field], np.ndarray)
- else np.asarray([features[field] for features in intermediate_features])
- )
- for field in feature_fields
- }
- cav_num = len(intermediate_features)
- model_input = {"intermediate_features": merged_feature_dict}
- else:
- typed_inputs: list[InferenceInput] = inference_input["inference_inputs"]
- if len(metadata) != len(typed_inputs):
- raise ValueError("Every inference input must have a matching metadata entry")
- model_input = {"processed_lidar": merge_inference_inputs(typed_inputs)}
- cav_num = len(typed_inputs)
-
- ego_sample = {
- **local_supervision,
- **model_input,
- "cav_num": cav_num,
- **self.__assemble_model_metadata(metadata, receive_frame),
- }
- if self.visualize:
- ego_sample.update(
- {
- "origin_lidar": np.vstack(visualization_context["projected_lidar_stack"]),
- "origin_lidar_by_agent": visualization_context["projected_lidar_stack"],
- "origin_lidar_roles": visualization_context["projected_lidar_roles"],
- "origin_lidar_agent_ids": visualization_context["projected_lidar_agent_ids"],
- }
- )
-
- return OrderedDict({"ego": ego_sample})
-
- def __getitem__(self, idx):
- base_data_dict = self.retrieve_base_data(idx, cur_ego_pose_flag=self.cur_ego_pose_flag)
- ego_id, ego_lidar_pose = self.__find_ego_vehicle(base_data_dict)
- visualization_base_data_dict = self.__retrieve_visualization_base_data(idx) if self.visualize else base_data_dict
-
- if self.payload_handler is not None:
- inference_input = self.__process_with_messages(ego_id, ego_lidar_pose, base_data_dict, idx)
- else:
- inference_input = self.__process_without_messages(ego_id, ego_lidar_pose, base_data_dict, idx)
-
- local_supervision = self.build_local_supervision(base_data_dict, ego_lidar_pose)
- visualization_context = self.build_visualization_context(ego_id, visualization_base_data_dict)
- return self.assemble_inference_sample(
- inference_input,
- local_supervision,
- visualization_context,
- idx,
- )
-
- def get_item_single_car(self, selected_cav_base, ego_pose):
- """
- Project and preprocess one agent's LiDAR input.
-
- Parameters
- ----------
- selected_cav_base : dict
- The dictionary contains a single CAV's raw information.
- ego_pose : list
- The ego vehicle lidar pose under world coordinate.
-
- Returns
- -------
- selected_cav_processed : dict
- The dictionary contains the cav's processed information.
- """
- selected_cav_processed = {}
-
- # calculate the transformation matrix
- transformation_matrix = selected_cav_base["params"]["transformation_matrix"]
-
- # filter lidar
- lidar_np = selected_cav_base["lidar_np"]
- lidar_np = shuffle_points(lidar_np)
- # remove points that hit itself
- lidar_np = mask_ego_points(lidar_np)
- # project the lidar to ego space
- if self.proj_first:
- lidar_np[:, :3] = box_utils.project_points_by_matrix_torch(lidar_np[:, :3], transformation_matrix)
- lidar_np = mask_points_by_range(lidar_np, self.params["preprocess"]["cav_lidar_range"])
- inference_input = build_inference_input(self.pre_processor.preprocess(lidar_np))
-
- selected_cav_processed.update(
- {
- "projected_lidar": lidar_np,
- "inference_input": inference_input,
- }
- )
-
- return selected_cav_processed
-
- @staticmethod
- def merge_features_to_dict(processed_feature_list):
- """
- Merge the preprocessed features from different cavs to the same
- dictionary.
-
- Parameters
- ----------
- processed_feature_list : list
- A list of dictionary containing all processed features from
- different cavs.
-
- Returns
- -------
- merged_feature_dict: dict
- key: feature names, value: list of features.
- """
-
- merged_feature_dict = OrderedDict()
-
- for i in range(len(processed_feature_list)):
- for feature_name, feature in processed_feature_list[i].items():
- if feature_name not in merged_feature_dict:
- merged_feature_dict[feature_name] = []
- if isinstance(feature, list):
- merged_feature_dict[feature_name] += feature
- else:
- merged_feature_dict[feature_name].append(feature)
-
- return merged_feature_dict
-
- def collate_batch_train(self, batch):
- # Intermediate fusion is different the other two
- output_dict = {"ego": {}}
-
- object_bbx_center = []
- object_bbx_mask = []
- object_ids = []
- processed_lidar_list = []
- intermediate_feature_list = []
- # used to record different scenario
- record_len = []
- label_dict_list = []
-
- # used for PriorEncoding for models
- velocity = []
- time_delay = []
- infra = []
-
- # pairwise transformation matrix
- pairwise_t_matrix_list = []
-
- # used for correcting the spatial transformation between delayed timestamp
- # and current timestamp
- spatial_correction_matrix_list = []
-
- if self.visualize:
- origin_lidar = []
-
- for i in range(len(batch)):
- ego_dict = batch[i]["ego"]
- object_bbx_center.append(ego_dict["object_bbx_center"])
- object_bbx_mask.append(ego_dict["object_bbx_mask"])
- object_ids.append(ego_dict["object_ids"])
-
- if "intermediate_features" in ego_dict:
- intermediate_feature_list.append(ego_dict["intermediate_features"])
- else:
- processed_lidar_list.append(ego_dict["processed_lidar"])
- record_len.append(ego_dict["cav_num"])
- label_dict_list.append(ego_dict["label_dict"])
-
- if self.model_name in PAIRWISE_METADATA_MODELS:
- pairwise_t_matrix_list.append(ego_dict["pairwise_t_matrix"])
- elif self.model_name in V2X_VIT_METADATA_MODELS:
- velocity.append(ego_dict["velocity"])
- time_delay.append(ego_dict["time_delay"])
- infra.append(ego_dict["infra"])
- spatial_correction_matrix_list.append(ego_dict["spatial_correction_matrix"])
-
- if self.visualize:
- origin_lidar.append(ego_dict["origin_lidar"])
- # convert to numpy, (B, max_num, 7)
- object_bbx_center = torch.from_numpy(np.array(object_bbx_center))
- object_bbx_mask = torch.from_numpy(np.array(object_bbx_mask))
-
- if intermediate_feature_list:
- if processed_lidar_list:
- raise TypeError("A batch cannot mix preprocessor inputs and intermediate features")
- intermediate_features_torch = {
- field: torch.from_numpy(np.concatenate([features[field] for features in intermediate_feature_list], axis=0))
- for field in intermediate_feature_list[0]
- }
- else:
- merged_feature_dict = self.merge_features_to_dict(processed_lidar_list)
- processed_lidar_torch_dict = self.pre_processor.collate_batch(merged_feature_dict)
- # [2, 3, 4, ..., M], M <= max_cav
- record_len = torch.from_numpy(np.array(record_len, dtype=int))
- label_torch_dict = self.post_processor.collate_batch(label_dict_list)
-
- # object id is only used during inference, where batch size is 1.
- # so here we only get the first element.
- output_dict["ego"].update(
- {
- "object_bbx_center": object_bbx_center,
- "object_bbx_mask": object_bbx_mask,
- "record_len": record_len,
- "label_dict": label_torch_dict,
- "object_ids": object_ids[0],
- }
- )
-
- if intermediate_feature_list:
- output_dict["ego"]["intermediate_features"] = intermediate_features_torch
- else:
- output_dict["ego"]["processed_lidar"] = processed_lidar_torch_dict
-
- if self.model_name in PAIRWISE_METADATA_MODELS:
- output_dict["ego"]["pairwise_t_matrix"] = torch.from_numpy(np.array(pairwise_t_matrix_list))
- elif self.model_name in V2X_VIT_METADATA_MODELS:
- velocity_tensor = torch.from_numpy(np.array(velocity))
- time_delay_tensor = torch.from_numpy(np.array(time_delay))
- infra_tensor = torch.from_numpy(np.array(infra))
- output_dict["ego"].update(
- {
- "prior_encoding": torch.stack([velocity_tensor, time_delay_tensor, infra_tensor], dim=-1).float(),
- "spatial_correction_matrix": torch.from_numpy(np.array(spatial_correction_matrix_list)),
- }
- )
-
- if self.visualize:
- origin_lidar = np.array(downsample_lidar_minimum(pcd_np_list=origin_lidar))
- origin_lidar = torch.from_numpy(origin_lidar)
- output_dict["ego"].update({"origin_lidar": origin_lidar})
-
- return output_dict
-
- def collate_batch_test(self, batch):
- assert len(batch) <= 1, "Batch size 1 is required during testing!"
- output_dict = self.collate_batch_train(batch)
-
- # check if anchor box in the batch
- if batch[0]["ego"]["anchor_box"] is not None:
- output_dict["ego"].update({"anchor_box": torch.from_numpy(np.array(batch[0]["ego"]["anchor_box"]))})
-
- if self.visualize and "origin_lidar_by_agent" in batch[0]["ego"]:
- output_dict["ego"].update(
- {
- "origin_lidar_by_agent": [torch.from_numpy(np.array(points)) for points in batch[0]["ego"]["origin_lidar_by_agent"]],
- "origin_lidar_roles": list(batch[0]["ego"]["origin_lidar_roles"]),
- "origin_lidar_agent_ids": list(batch[0]["ego"]["origin_lidar_agent_ids"]),
- }
- )
-
- # save the transformation matrix (4, 4) to ego vehicle
- transformation_matrix_torch = torch.from_numpy(np.identity(4)).float()
- output_dict["ego"].update({"transformation_matrix": transformation_matrix_torch})
-
- return output_dict
-
- def post_process(self, data_dict, output_dict):
- """
- Process the outputs of the model to 2D/3D bounding box.
-
- Parameters
- ----------
- data_dict : dict
- The dictionary containing the origin input data of model.
-
- output_dict :dict
- The dictionary containing the output of the model.
-
- Returns
- -------
- pred_box_tensor : torch.Tensor
- The tensor of prediction bounding box after NMS.
- gt_box_tensor : torch.Tensor
- The tensor of gt bounding box.
- """
- pred_box_tensor, pred_score = self.post_processor.post_process(data_dict, output_dict)
- gt_box_tensor = self.post_processor.generate_gt_bbx(data_dict)
-
- return pred_box_tensor, pred_score, gt_box_tensor
diff --git a/OpenCOOD/opencood/data_utils/datasets/intermediate_fusion_dataset_v2.py b/OpenCOOD/opencood/data_utils/datasets/intermediate_fusion_dataset_v2.py
deleted file mode 100644
index c21f2e72..00000000
--- a/OpenCOOD/opencood/data_utils/datasets/intermediate_fusion_dataset_v2.py
+++ /dev/null
@@ -1,595 +0,0 @@
-"""
-Dataset class for 2-stage backbone intermediate fusion
-"""
-
-import math
-from collections import OrderedDict
-from collections.abc import Callable
-from typing import Any
-
-import numpy as np
-import torch
-
-import opencood.data_utils.datasets
-import opencood.data_utils.post_processor as post_processor
-from opencood.utils import box_utils
-from opencood.data_utils.datasets import basedataset
-from opencood.models.communication_adapters import PoseFrameMetadata
-from opencood.data_utils.pre_processor import build_preprocessor
-from opencood.utils.pcd_utils import pcd_to_np, mask_points_by_range, mask_ego_points, shuffle_points
-from opencood.utils.transformation_utils import x1_to_x2
-from opencood.pcdet_utils.roiaware_pool3d.roiaware_pool3d_utils import points_in_boxes_cpu
-
-
-# TODO: The fpvrcnn_intermediate_fusion model has an issue with weights on this dataset
-# TODO: Verify the model behavior with this dataset
-# The weights appear to be incorrectly arranged
-# size mismatch for spconv_block.conv_out.0.weight:
-# copying a param with shape torch.Size([3, 1, 1, 64, 64]) from checkpoint,
-# while the current model expects torch.Size([64, 3, 1, 1, 64])
-# The model will need to be retrained and the code updated
-class IntermediateFusionDatasetV2(basedataset.BaseDataset):
- """
- This class is for intermediate fusion where each vehicle transmit the
- deep features to ego.
- """
-
- def __init__(self, params, visualize, train=True, payload_handler=None):
- super(IntermediateFusionDatasetV2, self).__init__(params, visualize, train)
- self.pre_processor = build_preprocessor(params["preprocess"], train)
- self.post_processor = post_processor.build_postprocessor(params["postprocess"], train)
-
- # whether there is a time delay between the time that cav project
- # lidar to ego and the ego receive the delivered feature
- if "cur_ego_pose_flag" in params["fusion"]["args"]:
- self.cur_ego_pose_flag = params["fusion"]["args"]["cur_ego_pose_flag"]
- else:
- self.cur_ego_pose_flag = True
-
- self.payload_handler = payload_handler
- self.module_name = "OpenCOOD.IntermediateFusionDatasetV2"
-
- def extract_data(
- self,
- idx: int,
- agent_payload_builder: Callable[[dict[str, Any], PoseFrameMetadata], object] | None = None,
- ) -> None:
- """
- Build model-specific payloads from local agent observations.
-
- Parameters
- ----------
- idx : int
- Dataset frame to publish.
- agent_payload_builder : Callable[[dict[str, Any], PoseFrameMetadata], object] | None
- Model adapter callback that converts one local agent observation
- and its metadata into a typed wire payload.
-
- Raises
- ------
- NotImplementedError
- If the configured model has no communication adapter for this
- dataset contract.
- """
- base_data_dict = self.retrieve_base_data(idx, cur_ego_pose_flag=self.cur_ego_pose_flag)
-
- if self.payload_handler is not None:
- if agent_payload_builder is None:
- raise NotImplementedError("IntermediateFusionDatasetV2 requires a model-specific communication adapter")
- for cav_id, selected_cav_base in base_data_dict.items():
- sender_pose = selected_cav_base["params"]["lidar_pose"]
- selected_cav_processed = self.build_agent_model_input(selected_cav_base, sender_pose)
- metadata = PoseFrameMetadata(
- lidar_pose=tuple(float(value) for value in sender_pose),
- capture_frame=int(idx - selected_cav_base["time_delay"]),
- )
- payload = agent_payload_builder(selected_cav_processed, metadata)
- self.payload_handler.set_opencda_payload(cav_id, self.module_name, payload)
-
- def __find_ego_vehicle(self, base_data_dict):
- ego_id = -1
- ego_lidar_pose = []
-
- # first find the ego vehicle's lidar pose
- for cav_id, cav_content in base_data_dict.items():
- if cav_content["ego"]:
- ego_id = cav_id
- ego_lidar_pose = cav_content["params"]["lidar_pose"]
- break
-
- assert cav_id == list(base_data_dict.keys())[0], "The first element in the OrderedDict must be ego"
- assert ego_id != -1
- assert len(ego_lidar_pose) > 0
-
- return ego_id, ego_lidar_pose
-
- def __retrieve_visualization_base_data(self, idx):
- scenario_index = 0
- for i, ele in enumerate(self.len_record):
- if idx < ele:
- scenario_index = i
- break
- scenario_database = self.scenario_database[scenario_index]
-
- timestamp_index = idx if scenario_index == 0 else idx - self.len_record[scenario_index - 1]
- timestamp_key = self.return_timestamp_key(scenario_database, timestamp_index)
- ego_cav_content = self.calc_dist_to_ego(scenario_database, timestamp_key)
-
- data = OrderedDict()
- for cav_id, cav_content in scenario_database.items():
- cav_snapshot = cav_content[timestamp_key]
- data[cav_id] = OrderedDict()
- data[cav_id]["ego"] = cav_content["ego"]
- data[cav_id]["time_delay"] = 0
- data[cav_id]["params"] = self.reform_param(
- cav_content,
- ego_cav_content,
- timestamp_key,
- timestamp_key,
- cur_ego_pose_flag=True,
- )
- data[cav_id]["lidar_np"] = cav_snapshot["lidar_np"] if "lidar_np" in cav_snapshot else pcd_to_np(cav_snapshot["lidar"])
- if "spoofing_mask" in cav_snapshot:
- data[cav_id]["spoofing_mask"] = cav_snapshot["spoofing_mask"]
- return data
-
- def __project_lidar_for_visualization(self, cav_base, ego_lidar_pose):
- transformation_matrix = x1_to_x2(cav_base["params"]["lidar_pose"], ego_lidar_pose)
- lidar_np = np.array(cav_base["lidar_np"], copy=True)
- lidar_np = mask_ego_points(lidar_np)
- lidar_np[:, :3] = box_utils.project_points_by_matrix_torch(lidar_np[:, :3], transformation_matrix)
- return mask_points_by_range(lidar_np, self.params["preprocess"]["cav_lidar_range"])
-
- def build_visualization_context(self, ego_id, ego_lidar_pose, base_data_dict):
- """
- Build local visualization data independently of wire delivery.
-
- Parameters
- ----------
- ego_id : str
- Identifier of the receiving agent.
- ego_lidar_pose : list[float]
- Receiver pose used to project visualization point clouds.
- base_data_dict : dict[str, Any]
- Complete local scene snapshot.
-
- Returns
- -------
- dict[str, Any]
- Per-agent point clouds, roles, and identifiers used only by
- visualization code.
- """
- if not self.visualize:
- return {
- "projected_lidar_stack": [],
- "projected_lidar_roles": [],
- "projected_lidar_agent_ids": [],
- }
-
- projected_lidar_stack = []
- projected_lidar_roles = []
- projected_lidar_agent_ids = []
-
- for cav_id in base_data_dict:
- projected_lidar_stack.append(self.__project_lidar_for_visualization(base_data_dict[cav_id], ego_lidar_pose))
- projected_lidar_roles.append("ego" if cav_id == ego_id else "other")
- projected_lidar_agent_ids.append(cav_id)
-
- return {
- "projected_lidar_stack": projected_lidar_stack,
- "projected_lidar_roles": projected_lidar_roles,
- "projected_lidar_agent_ids": projected_lidar_agent_ids,
- }
-
- def __process_with_messages(self, ego_id, ego_lidar_pose, base_data_dict, visualization_ego_lidar_pose, visualization_base_data_dict):
- processed_features = []
- model_lidar_stack = []
- remote_agent_outputs = []
-
- ego_cav_base = base_data_dict.get(ego_id)
- ego_cav_processed = self.build_agent_model_input(ego_cav_base, ego_lidar_pose)
-
- processed_features.append(ego_cav_processed["processed_features"])
- model_lidar_stack.append(ego_cav_processed["projected_lidar"])
-
- if self.communication_adapter is None:
- raise RuntimeError("IntermediateFusionDatasetV2 requires an attached communication adapter")
-
- if ego_id in self.payload_handler.current_artery_payload:
- for cav_id, _ in base_data_dict.items():
- if cav_id == ego_id:
- continue
- raw_payload = self.payload_handler.get_artery_payload(ego_id, cav_id, self.module_name)
- if raw_payload is None:
- continue
- remote_agent_outputs.append(self.communication_adapter.decode_received_payload(raw_payload, ego_lidar_pose))
-
- return {
- "processed_features": processed_features,
- "model_lidar_stack": model_lidar_stack,
- "remote_agent_outputs": remote_agent_outputs,
- **self.build_visualization_context(ego_id, visualization_ego_lidar_pose, visualization_base_data_dict),
- }
-
- def build_local_supervision(self, base_data_dict: dict[str, Any], ego_lidar_pose: list[float]) -> dict[str, Any]:
- """
- Build two-stage ground truth from the complete local scene.
-
- Parameters
- ----------
- base_data_dict : dict[str, Any]
- Complete local dataset snapshot, independent of packet delivery.
- ego_lidar_pose : list[float]
- Current receiver LiDAR pose.
-
- Returns
- -------
- dict[str, Any]
- Ground-truth boxes, masks, identifiers, and anchors, plus
- stage-two targets when building a training sample.
- """
- object_bbx_center, object_bbx_mask, object_ids = self.post_processor.generate_object_center(
- list(base_data_dict.values()),
- ego_lidar_pose,
- )
- anchor_box = self.post_processor.generate_anchor_box()
- supervision: dict[str, Any] = {
- "object_bbx_center": object_bbx_center,
- "object_bbx_mask": object_bbx_mask,
- "object_ids": object_ids,
- "anchor_box": anchor_box,
- }
- if self.payload_handler is None:
- supervision["stage2_label"] = self.post_processor.generate_label(
- gt_box_center=object_bbx_center,
- anchors=anchor_box,
- mask=object_bbx_mask,
- )
- return supervision
-
- def __process_without_messages(self, ego_id, ego_lidar_pose, base_data_dict, visualization_ego_lidar_pose, visualization_base_data_dict):
- processed_features = []
- object_stack = []
- model_lidar_stack = []
- agent_to_ego = []
-
- for cav_id, selected_cav_base in base_data_dict.items():
- dx = selected_cav_base["params"]["lidar_pose"][0] - ego_lidar_pose[0]
- dy = selected_cav_base["params"]["lidar_pose"][1] - ego_lidar_pose[1]
- distance = math.hypot(dx, dy)
-
- if distance > opencood.data_utils.datasets.COM_RANGE:
- continue
-
- agent_pose = selected_cav_base["params"]["lidar_pose"]
- selected_cav_processed = self.get_item_single_car(selected_cav_base, agent_pose)
-
- if len(selected_cav_processed["projected_lidar"]) > 10:
- object_stack.append(selected_cav_processed["object_bbx_center"])
- processed_features.append(selected_cav_processed["processed_features"])
- model_lidar_stack.append(selected_cav_processed["projected_lidar"])
- agent_to_ego.append(x1_to_x2(agent_pose, ego_lidar_pose))
-
- return {
- "processed_features": processed_features,
- "object_stack": object_stack,
- "model_lidar_stack": model_lidar_stack,
- "agent_to_ego": agent_to_ego,
- **self.build_visualization_context(ego_id, visualization_ego_lidar_pose, visualization_base_data_dict),
- }
-
- def __getitem__(self, idx):
- # put here to avoid initialization error
- base_data_dict = self.retrieve_base_data(idx, cur_ego_pose_flag=self.cur_ego_pose_flag)
-
- processed_data_dict = OrderedDict()
- processed_data_dict["ego"] = {}
-
- ego_id, ego_lidar_pose = self.__find_ego_vehicle(base_data_dict)
- visualization_base_data_dict = self.__retrieve_visualization_base_data(idx) if self.visualize else base_data_dict
- _, visualization_ego_lidar_pose = self.__find_ego_vehicle(visualization_base_data_dict)
-
- if self.payload_handler is not None:
- data = self.__process_with_messages(
- ego_id,
- ego_lidar_pose,
- base_data_dict,
- visualization_ego_lidar_pose,
- visualization_base_data_dict,
- )
- else:
- data = self.__process_without_messages(
- ego_id,
- ego_lidar_pose,
- base_data_dict,
- visualization_ego_lidar_pose,
- visualization_base_data_dict,
- )
-
- # merge preprocessed features from different cavs into the same dict
- cav_num = len(data["processed_features"])
- merged_feature_dict = self.merge_features_to_dict(data["processed_features"])
-
- local_supervision = self.build_local_supervision(base_data_dict, ego_lidar_pose)
- anchor_box = local_supervision["anchor_box"]
-
- processed_data_dict["ego"].update(
- {
- "object_bbx_center": local_supervision["object_bbx_center"],
- "object_bbx_mask": local_supervision["object_bbx_mask"],
- "object_ids": local_supervision["object_ids"],
- "anchor_box": anchor_box,
- "processed_lidar": merged_feature_dict,
- "cav_num": cav_num,
- }
- )
-
- if "stage2_label" in local_supervision:
- label_dict_no_coop = []
- for boxes, points in zip(data["object_stack"], data["model_lidar_stack"]):
- point_indices = points_in_boxes_cpu(points[:, :3], boxes[:, [0, 1, 2, 5, 4, 3, 6]])
- cur_mask = point_indices.sum(axis=1) > 0
- if cur_mask.sum() == 0:
- label_dict_no_coop.append(
- {
- "pos_equal_one": np.zeros((*anchor_box.shape[:2], self.post_processor.anchor_num)),
- "neg_equal_one": np.ones((*anchor_box.shape[:2], self.post_processor.anchor_num)),
- "targets": np.zeros((*anchor_box.shape[:2], self.post_processor.anchor_num * 7)),
- }
- )
- continue
- bbx_center = np.zeros((self.params["postprocess"]["max_num"], 7))
- bbx_mask = np.zeros(self.params["postprocess"]["max_num"])
- bbx_center[: boxes[cur_mask].shape[0], :] = boxes[cur_mask]
- bbx_mask[: boxes[cur_mask].shape[0]] = 1
- label_dict_no_coop.append(
- self.post_processor.generate_label(
- gt_box_center=bbx_center,
- anchors=anchor_box,
- mask=bbx_mask,
- )
- )
- processed_data_dict["ego"]["label_dict"] = {
- "stage1": label_dict_no_coop,
- "stage2": local_supervision["stage2_label"],
- }
-
- processed_data_dict["ego"].update(
- {
- "origin_lidar": data["model_lidar_stack"],
- "origin_lidar_by_agent": data["projected_lidar_stack"],
- "origin_lidar_roles": data["projected_lidar_roles"],
- "origin_lidar_agent_ids": data["projected_lidar_agent_ids"],
- }
- )
- if "remote_agent_outputs" in data:
- processed_data_dict["ego"]["remote_agent_outputs"] = data["remote_agent_outputs"]
- if "agent_to_ego" in data:
- processed_data_dict["ego"]["agent_to_ego"] = data["agent_to_ego"]
- return processed_data_dict
-
- def build_agent_model_input(self, selected_cav_base, target_pose):
- """
- Build one agent's local two-stage input without supervision data.
-
- Parameters
- ----------
- selected_cav_base : dict
- Raw and metadata fields for one agent.
- target_pose : list
- Pose of the coordinate frame used for this model input.
-
- Returns
- -------
- dict
- Projected raw points and their voxelized representation.
- """
- transformation_matrix = x1_to_x2(selected_cav_base["params"]["lidar_pose"], target_pose)
-
- lidar_np = selected_cav_base["lidar_np"]
- lidar_np = shuffle_points(lidar_np)
- lidar_np = mask_ego_points(lidar_np)
- lidar_np[:, :3] = box_utils.project_points_by_matrix_torch(lidar_np[:, :3], transformation_matrix)
- lidar_np = mask_points_by_range(lidar_np, self.params["preprocess"]["cav_lidar_range"])
- return {
- "projected_lidar": lidar_np,
- "processed_features": self.pre_processor.preprocess(lidar_np),
- }
-
- def get_item_single_car(self, selected_cav_base, ego_pose):
- """
- Build one local training agent with its stage-one ground truth.
-
- Parameters
- ----------
- selected_cav_base : dict
- Raw and metadata fields for one agent.
- ego_pose : list
- Pose of the coordinate frame used for model input and targets.
-
- Returns
- -------
- dict
- Agent model input plus local stage-one ground-truth boxes.
- """
- selected_cav_processed = self.build_agent_model_input(selected_cav_base, ego_pose)
- object_bbx_center, object_bbx_mask, _ = self.post_processor.generate_object_center([selected_cav_base], ego_pose)
- selected_cav_processed["object_bbx_center"] = object_bbx_center[object_bbx_mask == 1]
- return selected_cav_processed
-
- @staticmethod
- def merge_features_to_dict(processed_feature_list):
- """
- Merge the preprocessed features from different cavs to the same
- dictionary.
-
- Parameters
- ----------
- processed_feature_list : list
- A list of dictionary containing all processed features from
- different cavs.
-
- Returns
- -------
- merged_feature_dict: dict
- key: feature names, value: list of features.
- """
-
- merged_feature_dict = OrderedDict()
-
- for i in range(len(processed_feature_list)):
- for feature_name, feature in processed_feature_list[i].items():
- if feature_name not in merged_feature_dict:
- merged_feature_dict[feature_name] = []
- if isinstance(feature, list):
- merged_feature_dict[feature_name] += feature
- else:
- merged_feature_dict[feature_name].append(feature)
-
- return merged_feature_dict
-
- def collate_batch_train(self, batch):
- # Intermediate fusion is different the other two
- output_dict = {"ego": {}}
-
- object_bbx_center = []
- object_bbx_mask = []
- object_ids = []
- processed_lidar_list = []
- # used to record different scenario
- record_len = []
- label_dict_list = []
- label_dict_no_coop_list = []
- has_labels = "label_dict" in batch[0]["ego"]
-
- origin_lidar = []
- agent_to_ego = []
-
- # added by yys, fpvrcnn needs anchors for
- # first stage proposal generation
- if batch[0]["ego"]["anchor_box"] is not None:
- output_dict["ego"].update({"anchor_box": torch.from_numpy(np.array(batch[0]["ego"]["anchor_box"]))})
-
- for i in range(len(batch)):
- ego_dict = batch[i]["ego"]
- object_bbx_center.append(ego_dict["object_bbx_center"])
- object_bbx_mask.append(ego_dict["object_bbx_mask"])
- object_ids.append(ego_dict["object_ids"])
-
- processed_lidar_list.append(ego_dict["processed_lidar"])
- record_len.append(ego_dict["cav_num"])
- if has_labels:
- label_dict_no_coop_list.append(ego_dict["label_dict"]["stage1"])
- label_dict_list.append(ego_dict["label_dict"]["stage2"])
-
- origin_lidar.append(ego_dict["origin_lidar"])
- if "agent_to_ego" in ego_dict:
- agent_to_ego.extend(ego_dict["agent_to_ego"])
-
- # convert to numpy, (B, max_num, 7)
- object_bbx_center = torch.from_numpy(np.array(object_bbx_center))
- object_bbx_mask = torch.from_numpy(np.array(object_bbx_mask))
-
- # example: {'voxel_features':[np.array([1,2,3]]),
- # np.array([3,5,6]), ...]}
- merged_feature_dict = self.merge_features_to_dict(processed_lidar_list)
- processed_lidar_torch_dict = self.pre_processor.collate_batch(merged_feature_dict)
- # [2, 3, 4, ..., M], M <= 5
- record_len = torch.from_numpy(np.array(record_len, dtype=int))
- # object id is only used during inference, where batch size is 1.
- # so here we only get the first element.
- output_dict["ego"].update(
- {
- "object_bbx_center": object_bbx_center,
- "object_bbx_mask": object_bbx_mask,
- "processed_lidar": processed_lidar_torch_dict,
- "record_len": record_len,
- "object_ids": object_ids[0],
- }
- )
- if has_labels:
- label_torch_dict = self.post_processor.collate_batch(label_dict_list)
- label_dict_no_coop = [label_dict for label_list in label_dict_no_coop_list for label_dict in label_list]
- output_dict["ego"]["label_dict"] = {
- "stage1": self.post_processor.collate_batch(label_dict_no_coop),
- "stage2": label_torch_dict,
- }
-
- coords = []
- idx = 0
- for b in range(len(batch)):
- for points in origin_lidar[b]:
- assert len(points) != 0
- coor_pad = np.pad(points, ((0, 0), (1, 0)), mode="constant", constant_values=idx)
- coords.append(coor_pad)
- idx += 1
- origin_lidar = np.concatenate(coords, axis=0)
-
- origin_lidar = torch.from_numpy(origin_lidar)
- output_dict["ego"].update({"origin_lidar": origin_lidar})
-
- if agent_to_ego:
- output_dict["ego"]["agent_to_ego"] = torch.from_numpy(np.asarray(agent_to_ego)).float()
-
- if "remote_agent_outputs" in batch[0]["ego"]:
- if len(batch) != 1:
- raise ValueError("Distributed two-stage inference requires batch size one")
- output_dict["ego"]["remote_agent_outputs"] = [
- {field: torch.from_numpy(np.asarray(value)) for field, value in agent_output.items()}
- for agent_output in batch[0]["ego"]["remote_agent_outputs"]
- ]
-
- return output_dict
-
- def collate_batch_test(self, batch):
- assert len(batch) <= 1, "Batch size 1 is required during testing!"
- output_dict = self.collate_batch_train(batch)
-
- # check if anchor box in the batch
- if batch[0]["ego"]["anchor_box"] is not None:
- output_dict["ego"].update({"anchor_box": torch.from_numpy(np.array(batch[0]["ego"]["anchor_box"]))})
-
- if "origin_lidar_by_agent" in batch[0]["ego"]:
- output_dict["ego"].update(
- {
- "origin_lidar_by_agent": [torch.from_numpy(np.array(points)) for points in batch[0]["ego"]["origin_lidar_by_agent"]],
- "origin_lidar_roles": list(batch[0]["ego"]["origin_lidar_roles"]),
- "origin_lidar_agent_ids": list(batch[0]["ego"]["origin_lidar_agent_ids"]),
- }
- )
-
- # save the transformation matrix (4, 4) to ego vehicle
- transformation_matrix_torch = torch.from_numpy(np.identity(4)).float()
- output_dict["ego"].update({"transformation_matrix": transformation_matrix_torch})
-
- return output_dict
-
- def post_process(self, data_dict, output_dict):
- """
- Process the outputs of the model to 2D/3D bounding box.
-
- Parameters
- ----------
- data_dict : dict
- The dictionary containing the origin input data of model.
-
- output_dict :dict
- The dictionary containing the output of the model.
-
- Returns
- -------
- pred_box_tensor : torch.Tensor
- The tensor of prediction bounding box after NMS.
- gt_box_tensor : torch.Tensor
- The tensor of gt bounding box.
- """
- pred_box_tensor, pred_score = self.post_processor.post_process(data_dict, output_dict)
- gt_box_tensor = self.post_processor.generate_gt_bbx(data_dict)
-
- return pred_box_tensor, pred_score, gt_box_tensor
-
- def visualize_result(self, pred_box_tensor, gt_tensor, pcd, show_vis, save_path, dataset=None):
- # we need to convert the pcd from [n, 5] -> [n, 4]
- pcd = pcd[:, 1:]
- # visualize the model output
- self.post_processor.visualize(pred_box_tensor, gt_tensor, pcd, show_vis, save_path, dataset=dataset)
diff --git a/OpenCOOD/opencood/data_utils/datasets/late_fusion_dataset.py b/OpenCOOD/opencood/data_utils/datasets/late_fusion_dataset.py
deleted file mode 100644
index 9a4b0464..00000000
--- a/OpenCOOD/opencood/data_utils/datasets/late_fusion_dataset.py
+++ /dev/null
@@ -1,566 +0,0 @@
-"""
-Dataset class for late fusion
-"""
-
-import math
-import random
-import logging
-from collections import OrderedDict
-from collections.abc import Callable
-from typing import Any
-
-import numpy as np
-import numpy.typing as npt
-import torch
-
-import opencood.data_utils.datasets
-from opencood.data_utils.post_processor import build_postprocessor
-from opencood.data_utils.datasets import basedataset
-from opencood.models.communication_adapters import (
- InferenceInput,
- PoseFrameMetadata,
- build_inference_input,
- inference_input_to_dict,
-)
-from opencood.data_utils.pre_processor import build_preprocessor
-from opencood.utils import box_utils
-from opencood.utils.pcd_utils import pcd_to_np, mask_points_by_range, mask_ego_points, shuffle_points, downsample_lidar_minimum
-from opencood.utils.transformation_utils import x1_to_x2
-
-logger = logging.getLogger("cavise.opencda.OpenCOOD.opencood.data_utils.datasets.late_fusion_dataset")
-
-
-class LateFusionDataset(basedataset.BaseDataset):
- """
- This class is for late fusion where each vehicle transmits its
- detection outputs to ego.
- """
-
- def __init__(self, params, visualize, train=True, payload_handler=None):
- super(LateFusionDataset, self).__init__(params, visualize, train)
- self.pre_processor = build_preprocessor(params["preprocess"], train)
- self.post_processor = build_postprocessor(params["postprocess"], train)
-
- self.payload_handler = payload_handler
- self.module_name = "OpenCOOD.LateFusionDataset"
-
- def __getitem__(self, idx):
- base_data_dict = self.retrieve_base_data(idx)
- if self.train:
- reformat_data_dict = self.get_item_train(base_data_dict)
- else:
- reformat_data_dict = self.get_item_test(idx, base_data_dict)
-
- return reformat_data_dict
-
- def extract_data(
- self,
- idx: int,
- agent_payload_builder: Callable[[InferenceInput, PoseFrameMetadata], object],
- ) -> None:
- """
- Publish local detector inputs through the late-fusion adapter.
-
- Parameters
- ----------
- idx : int
- Dataset frame to publish.
- agent_payload_builder : Callable
- Adapter callback that runs the sender-side detector and builds its
- wire representation.
- """
- base_data_dict = self.retrieve_base_data(idx)
-
- if self.payload_handler is not None:
- for cav_id, selected_cav_base in base_data_dict.items():
- selected_cav_processed = self.__build_model_input(selected_cav_base)
- payload = agent_payload_builder(
- selected_cav_processed["inference_input"],
- PoseFrameMetadata(
- lidar_pose=selected_cav_base["params"]["lidar_pose"],
- capture_frame=idx - int(selected_cav_base["time_delay"]),
- ),
- )
- self.payload_handler.set_opencda_payload(cav_id, self.module_name, payload)
-
- def __find_ego_vehicle(self, base_data_dict):
- ego_id = -1
- ego_lidar_pose = []
-
- # first find the ego vehicle's lidar pose
- for cav_id, cav_content in base_data_dict.items():
- if cav_content["ego"]:
- ego_id = cav_id
- ego_lidar_pose = cav_content["params"]["lidar_pose"]
- break
-
- assert cav_id == list(base_data_dict.keys())[0], "The first element in the OrderedDict must be ego"
- assert ego_id != -1
-
- return ego_id, ego_lidar_pose
-
- def __retrieve_visualization_base_data(self, idx):
- scenario_index = 0
- for i, ele in enumerate(self.len_record):
- if idx < ele:
- scenario_index = i
- break
- scenario_database = self.scenario_database[scenario_index]
-
- timestamp_index = idx if scenario_index == 0 else idx - self.len_record[scenario_index - 1]
- timestamp_key = self.return_timestamp_key(scenario_database, timestamp_index)
- ego_cav_content = self.calc_dist_to_ego(scenario_database, timestamp_key)
-
- data = OrderedDict()
- for cav_id, cav_content in scenario_database.items():
- cav_snapshot = cav_content[timestamp_key]
- data[cav_id] = OrderedDict()
- data[cav_id]["ego"] = cav_content["ego"]
- data[cav_id]["time_delay"] = 0
- data[cav_id]["params"] = self.reform_param(
- cav_content,
- ego_cav_content,
- timestamp_key,
- timestamp_key,
- cur_ego_pose_flag=True,
- )
- data[cav_id]["lidar_np"] = cav_snapshot["lidar_np"] if "lidar_np" in cav_snapshot else pcd_to_np(cav_snapshot["lidar"])
- if "spoofing_mask" in cav_snapshot:
- data[cav_id]["spoofing_mask"] = cav_snapshot["spoofing_mask"]
- return data
-
- def __project_lidar_for_visualization(self, cav_base, ego_lidar_pose):
- transformation_matrix = x1_to_x2(cav_base["params"]["lidar_pose"], ego_lidar_pose)
- lidar_np = np.array(cav_base["lidar_np"], copy=True)
- lidar_np = mask_ego_points(lidar_np)
- lidar_np[:, :3] = box_utils.project_points_by_matrix_torch(lidar_np[:, :3], transformation_matrix)
- return mask_points_by_range(lidar_np, self.params["preprocess"]["cav_lidar_range"])
-
- def build_visualization_context(
- self,
- ego_id: str,
- ego_lidar_pose: list[float],
- base_data_dict: dict[str, Any],
- ) -> dict[str, Any]:
- """
- Build visualization data from the local scene snapshot.
-
- Parameters
- ----------
- ego_id : str
- Identifier of the receiving ego agent.
- ego_lidar_pose : list[float]
- Current ego LiDAR pose in world coordinates.
- base_data_dict : dict[str, Any]
- Local dataset snapshot, independent of delivered messages.
-
- Returns
- -------
- dict[str, Any]
- Per-agent projected point clouds and their local identities.
- """
- if not self.visualize:
- return {}
-
- origin_lidar_by_agent = []
- origin_lidar_roles = []
- origin_lidar_agent_ids = []
-
- for cav_id in base_data_dict:
- origin_lidar_by_agent.append(self.__project_lidar_for_visualization(base_data_dict[cav_id], ego_lidar_pose))
- origin_lidar_roles.append("ego" if cav_id == ego_id else "other")
- origin_lidar_agent_ids.append(cav_id)
-
- return {
- "origin_lidar_by_agent": origin_lidar_by_agent,
- "origin_lidar_roles": origin_lidar_roles,
- "origin_lidar_agent_ids": origin_lidar_agent_ids,
- }
-
- def __process_with_messages(self, ego_id, ego_lidar_pose, base_data_dict):
- processed_data_dict = OrderedDict()
-
- ego_cav_base = base_data_dict.get(ego_id)
- ego_cav_processed = self.__build_model_input(ego_cav_base)
-
- transformation_matrix_info = x1_to_x2(ego_lidar_pose, ego_lidar_pose)
- processed_data_dict["ego"] = {
- "inference_input": ego_cav_processed["inference_input"],
- "transformation_matrix": transformation_matrix_info,
- }
-
- if ego_id in self.payload_handler.current_artery_payload:
- for cav_id, _ in base_data_dict.items():
- if cav_id == ego_id:
- continue
- raw_payload = self.payload_handler.get_artery_payload(ego_id, cav_id, self.module_name)
- if raw_payload is None:
- continue
- if self.communication_adapter is None:
- raise RuntimeError("Late-fusion payload decoding requires a communication adapter")
- processed_data_dict[cav_id] = self.communication_adapter.decode_received_payload(
- raw_payload,
- ego_lidar_pose,
- )
-
- return processed_data_dict
-
- def __process_without_messages(self, ego_id, ego_lidar_pose, base_data_dict):
- processed_data_dict = OrderedDict()
-
- for cav_id, selected_cav_base in base_data_dict.items():
- dx = selected_cav_base["params"]["lidar_pose"][0] - ego_lidar_pose[0]
- dy = selected_cav_base["params"]["lidar_pose"][1] - ego_lidar_pose[1]
- distance = math.hypot(dx, dy)
-
- if distance > opencood.data_utils.datasets.COM_RANGE:
- continue
-
- # find the transformation matrix from current cav to ego.
- cav_lidar_pose = selected_cav_base["params"]["lidar_pose"]
- transformation_matrix = x1_to_x2(cav_lidar_pose, ego_lidar_pose)
-
- selected_cav_processed = self.__build_model_input(selected_cav_base)
- update_cav = "ego" if cav_id == ego_id else cav_id
- processed_data_dict[update_cav] = {
- "inference_input": selected_cav_processed["inference_input"],
- "transformation_matrix": transformation_matrix,
- }
-
- return processed_data_dict
-
- def build_local_supervision(self, base_data_dict: dict[str, Any], ego_lidar_pose: list[float]) -> dict[str, Any]:
- """
- Build ego-frame ground truth and labels from the local scene.
-
- Parameters
- ----------
- base_data_dict : dict[str, Any]
- Complete local dataset snapshot for the current scene.
- ego_lidar_pose : list[float]
- Current ego LiDAR pose in world coordinates.
-
- Returns
- -------
- dict[str, Any]
- Ego-frame ground truth, anchors, and target labels.
- """
- object_bbx_center, object_bbx_mask, object_ids = self.post_processor.generate_object_center(list(base_data_dict.values()), ego_lidar_pose)
- anchor_box = self.post_processor.generate_anchor_box()
- label_dict = self.post_processor.generate_label(
- gt_box_center=object_bbx_center,
- anchors=anchor_box,
- mask=object_bbx_mask,
- )
- return {
- "object_bbx_center": object_bbx_center,
- "object_bbx_mask": object_bbx_mask,
- "object_ids": object_ids,
- "anchor_box": anchor_box,
- "label_dict": label_dict,
- }
-
- def assemble_inference_sample(
- self,
- inference_input: OrderedDict[str, dict[str, Any]],
- local_supervision: dict[str, Any],
- visualization_context: dict[str, Any],
- ) -> OrderedDict[str, dict[str, Any]]:
- """
- Combine delivered model inputs with local-only dataset data.
-
- Parameters
- ----------
- inference_input : collections.OrderedDict
- Ego model input plus model inputs from delivered messages.
- local_supervision : dict[str, Any]
- Ground truth and labels produced from the local scene.
- visualization_context : dict[str, Any]
- Visualization data produced from the local scene.
-
- Returns
- -------
- collections.OrderedDict
- Late-fusion sample ready for collation.
- """
- anchor_box = local_supervision["anchor_box"]
- for cav_content in inference_input.values():
- if "inference_input" in cav_content:
- cav_content["anchor_box"] = anchor_box
-
- inference_input["ego"].update(local_supervision)
- inference_input["ego"].update(visualization_context)
- return inference_input
-
- def __prepare_lidar(self, selected_cav_base: dict[str, Any]) -> npt.NDArray[Any]:
- """
- Filter one agent's local LiDAR point cloud.
-
- Parameters
- ----------
- selected_cav_base : dict[str, Any]
- Raw dataset entry for one agent.
-
- Returns
- -------
- numpy.ndarray
- Shuffled and range-filtered LiDAR points.
- """
- lidar_np = shuffle_points(selected_cav_base["lidar_np"])
- lidar_np = mask_points_by_range(lidar_np, self.params["preprocess"]["cav_lidar_range"])
- return mask_ego_points(lidar_np)
-
- def __build_model_input(self, selected_cav_base: dict[str, Any]) -> dict[str, Any]:
- """
- Build model input without constructing local supervision.
-
- Parameters
- ----------
- selected_cav_base : dict[str, Any]
- Raw dataset entry for one agent.
-
- Returns
- -------
- dict[str, Any]
- Preprocessed LiDAR features used by the detector.
- """
- lidar_np = self.__prepare_lidar(selected_cav_base)
- return {"inference_input": build_inference_input(self.pre_processor.preprocess(lidar_np))}
-
- def get_item_single_car(self, selected_cav_base):
- """
- Process a single CAV's information for the train/test pipeline.
-
- Parameters
- ----------
- selected_cav_base : dict
- The dictionary contains a single CAV's raw information.
-
- Returns
- -------
- selected_cav_processed : dict
- The dictionary contains the cav's processed information.
- """
- selected_cav_processed = {}
-
- lidar_np = self.__prepare_lidar(selected_cav_base)
-
- # generate the bounding box(n, 7) under the cav's space
- object_bbx_center, object_bbx_mask, object_ids = self.post_processor.generate_object_center(
- [selected_cav_base], selected_cav_base["params"]["lidar_pose"]
- )
- # data augmentation
- lidar_np, object_bbx_center, object_bbx_mask = self.augment(lidar_np, object_bbx_center, object_bbx_mask)
-
- if self.visualize:
- selected_cav_processed.update({"origin_lidar": lidar_np})
-
- # pre-process the lidar to voxel/bev/downsampled lidar
- lidar_dict = self.pre_processor.preprocess(lidar_np)
- selected_cav_processed.update({"processed_lidar": lidar_dict})
-
- # generate the anchor boxes
- anchor_box = self.post_processor.generate_anchor_box()
- selected_cav_processed.update({"anchor_box": anchor_box})
-
- selected_cav_processed.update({"object_bbx_center": object_bbx_center, "object_bbx_mask": object_bbx_mask, "object_ids": object_ids})
-
- # generate targets label
- label_dict = self.post_processor.generate_label(gt_box_center=object_bbx_center, anchors=anchor_box, mask=object_bbx_mask)
- selected_cav_processed.update({"label_dict": label_dict})
-
- return selected_cav_processed
-
- def get_item_train(self, base_data_dict):
- processed_data_dict = OrderedDict()
-
- # during training, we return a random cav's data
- if not self.visualize:
- _, selected_cav_base = random.choice(list(base_data_dict.items()))
- else:
- _, selected_cav_base = list(base_data_dict.items())[0]
-
- selected_cav_processed = self.get_item_single_car(selected_cav_base)
- processed_data_dict.update({"ego": selected_cav_processed})
-
- return processed_data_dict
-
- def get_item_test(self, idx, base_data_dict):
- ego_id = -1
- ego_lidar_pose = []
-
- ego_id, ego_lidar_pose = self.__find_ego_vehicle(base_data_dict)
- visualization_base_data_dict = self.__retrieve_visualization_base_data(idx) if self.visualize else base_data_dict
- _, visualization_ego_lidar_pose = self.__find_ego_vehicle(visualization_base_data_dict)
-
- if self.payload_handler is not None:
- inference_input = self.__process_with_messages(ego_id, ego_lidar_pose, base_data_dict)
- else:
- inference_input = self.__process_without_messages(ego_id, ego_lidar_pose, base_data_dict)
-
- local_supervision = self.build_local_supervision(base_data_dict, ego_lidar_pose)
- visualization_context = self.build_visualization_context(
- ego_id,
- visualization_ego_lidar_pose,
- visualization_base_data_dict,
- )
- return self.assemble_inference_sample(inference_input, local_supervision, visualization_context)
-
- def __collate_processed_lidar(self, cav_content):
- if "inference_input" in cav_content:
- return self.pre_processor.collate_batch([inference_input_to_dict(cav_content["inference_input"])])
- return self.pre_processor.collate_batch([cav_content["processed_lidar"]])
-
- def collate_batch_test(self, batch):
- """
- Customized collate function for pytorch dataloader during testing
- for late fusion dataset.
-
- Parameters
- ----------
- batch : dict
-
- Returns
- -------
- batch : dict
- Reformatted batch.
- """
- # currently, we only support batch size of 1 during testing
- assert len(batch) <= 1, "Batch size 1 is required during testing!"
- batch = batch[0]
-
- output_dict = {}
-
- # for late fusion, we also need to stack the lidar for better
- # visualization
- if self.visualize:
- projected_lidar_list = []
- origin_lidar_by_agent = batch.get("ego", {}).get("origin_lidar_by_agent")
- origin_lidar_roles = batch.get("ego", {}).get("origin_lidar_roles", [])
- origin_lidar_agent_ids = batch.get("ego", {}).get("origin_lidar_agent_ids", [])
-
- for cav_id, cav_content in batch.items():
- output_dict.update({cav_id: {}})
-
- if "pred_box_tensor" in cav_content:
- output_dict[cav_id].update(
- {
- "pred_box_tensor": torch.from_numpy(np.asarray(cav_content["pred_box_tensor"])),
- "pred_score": torch.from_numpy(np.asarray(cav_content["pred_score"])),
- }
- )
- continue
-
- # the anchor box is the same for all bounding boxes usually, thus
- # we don't need the batch dimension.
- if cav_content["anchor_box"] is not None:
- output_dict[cav_id].update({"anchor_box": torch.from_numpy(np.array(cav_content["anchor_box"]))})
- if self.visualize:
- if "origin_lidar" in cav_content:
- transformation_matrix = cav_content["transformation_matrix"]
- local_lidar = np.array(cav_content["origin_lidar"], copy=True)
- projected_lidar = np.array(cav_content["origin_lidar"], copy=True)
- projected_lidar[:, :3] = box_utils.project_points_by_matrix_torch(projected_lidar[:, :3], transformation_matrix)
- projected_lidar_list.append(projected_lidar)
-
- # processed lidar dictionary
- processed_lidar_torch_dict = self.__collate_processed_lidar(cav_content)
-
- # save the transformation matrix (4, 4) to ego vehicle
- transformation_matrix_torch = torch.from_numpy(np.array(cav_content["transformation_matrix"])).float()
-
- output_dict[cav_id].update(
- {
- "processed_lidar": processed_lidar_torch_dict,
- "batch_size": 1,
- "transformation_matrix": transformation_matrix_torch,
- }
- )
-
- if "object_bbx_center" in cav_content:
- output_dict[cav_id].update(
- {
- "object_bbx_center": torch.from_numpy(np.array([cav_content["object_bbx_center"]])),
- "object_bbx_mask": torch.from_numpy(np.array([cav_content["object_bbx_mask"]])),
- "label_dict": self.post_processor.collate_batch([cav_content["label_dict"]]),
- "object_ids": cav_content["object_ids"],
- }
- )
-
- if self.visualize:
- if "origin_lidar" in cav_content:
- origin_lidar = np.array(downsample_lidar_minimum(pcd_np_list=[local_lidar]))
- origin_lidar = torch.from_numpy(origin_lidar)
- output_dict[cav_id].update({"origin_lidar": origin_lidar, "origin_lidar_local": origin_lidar.clone(), "agent_id": cav_id})
-
- if self.visualize:
- if origin_lidar_by_agent is not None:
- output_dict["ego"].update(
- {
- "origin_lidar": torch.from_numpy(np.vstack(origin_lidar_by_agent)),
- "origin_lidar_by_agent": [torch.from_numpy(np.array(points)) for points in origin_lidar_by_agent],
- "origin_lidar_roles": list(origin_lidar_roles),
- "origin_lidar_agent_ids": list(origin_lidar_agent_ids),
- }
- )
- elif projected_lidar_list:
- projected_lidar_stack = torch.from_numpy(np.vstack(projected_lidar_list))
- output_dict["ego"].update({"origin_lidar": projected_lidar_stack})
-
- return output_dict
-
- def post_process(self, data_dict, output_dict):
- """
- Process the outputs of the model to 2D/3D bounding box.
-
- Parameters
- ----------
- data_dict : dict
- The dictionary containing the origin input data of model.
-
- output_dict :dict
- The dictionary containing the output of the model.
-
- Returns
- -------
- pred_box_tensor : torch.Tensor
- The tensor of prediction bounding box after NMS.
- gt_box_tensor : torch.Tensor
- The tensor of gt bounding box.
- """
- pred_box_list = []
- pred_score_list = []
- for cav_id, cav_content in data_dict.items():
- if "pred_box_tensor" in cav_content:
- boxes = cav_content["pred_box_tensor"]
- scores = cav_content["pred_score"]
- else:
- boxes, scores = self.post_processor.decode_agent_predictions(
- cav_content,
- output_dict[cav_id],
- )
- if boxes.shape[0] > 0:
- transformation_matrix = cav_content["transformation_matrix"]
- if boxes.shape[-1] == 3:
- boxes = box_utils.project_box3d(boxes, transformation_matrix)
- else:
- box3d = torch.nn.functional.pad(boxes, (0, 1))
- boxes = box_utils.project_points_by_matrix_torch(
- box3d.reshape(-1, 3),
- transformation_matrix,
- )[:, :2].reshape(-1, 4, 2)
-
- if boxes.shape[0] > 0:
- pred_box_list.append(boxes)
- pred_score_list.append(scores)
-
- if pred_box_list:
- pred_box_tensor, pred_score = self.post_processor.post_process_detections(
- torch.cat(pred_box_list, dim=0),
- torch.cat(pred_score_list, dim=0),
- )
- else:
- pred_box_tensor, pred_score = None, None
- gt_box_tensor = self.post_processor.generate_gt_bbx({"ego": data_dict["ego"]})
-
- return pred_box_tensor, pred_score, gt_box_tensor
diff --git a/OpenCOOD/opencood/data_utils/post_processor/__init__.py b/OpenCOOD/opencood/data_utils/post_processor/__init__.py
deleted file mode 100644
index da18532c..00000000
--- a/OpenCOOD/opencood/data_utils/post_processor/__init__.py
+++ /dev/null
@@ -1,19 +0,0 @@
-from opencood.data_utils.post_processor.voxel_postprocessor import VoxelPostprocessor
-from opencood.data_utils.post_processor.bev_postprocessor import BevPostprocessor
-from opencood.data_utils.post_processor.ciassd_postprocessor import CiassdPostprocessor
-from opencood.data_utils.post_processor.fpvrcnn_postprocessor import FpvrcnnPostprocessor
-
-__all__ = {
- "VoxelPostprocessor": VoxelPostprocessor,
- "BevPostprocessor": BevPostprocessor,
- "CiassdPostprocessor": CiassdPostprocessor,
- "FpvrcnnPostprocessor": FpvrcnnPostprocessor,
-}
-
-
-def build_postprocessor(anchor_cfg, train):
- process_method_name = anchor_cfg["core_method"]
- assert process_method_name in ["VoxelPostprocessor", "BevPostprocessor", "CiassdPostprocessor", "FpvrcnnPostprocessor"]
- anchor_generator = __all__[process_method_name](anchor_params=anchor_cfg, train=train)
-
- return anchor_generator
diff --git a/OpenCOOD/opencood/data_utils/post_processor/base_postprocessor.py b/OpenCOOD/opencood/data_utils/post_processor/base_postprocessor.py
deleted file mode 100644
index c89aa6d9..00000000
--- a/OpenCOOD/opencood/data_utils/post_processor/base_postprocessor.py
+++ /dev/null
@@ -1,129 +0,0 @@
-"""
-Template for AnchorGenerator
-"""
-
-import numpy as np
-import torch
-
-from opencood.utils import box_utils
-
-
-class BasePostprocessor(object):
- """
- Template for Anchor generator.
-
- Parameters
- ----------
- anchor_params : dict
- The dictionary containing all anchor-related parameters.
- train : bool
- Indicate train or test mode.
-
- Attributes
- ----------
- bbx_dict : dictionary
- Contain all objects information across the cav, key: id, value: bbx
- coordinates (1, 7)
- """
-
- def __init__(self, anchor_params, train=True):
- self.params = anchor_params
- self.bbx_dict = {} # noqa: DC05
- self.train = train
-
- def generate_anchor_box(self):
- # needs to be overloaded
- return None
-
- def generate_label(self, *argv):
- return None
-
- def generate_gt_bbx(self, data_dict):
- """
- The base postprocessor will generate 3d groundtruth bounding box.
-
- Parameters
- ----------
- data_dict : dict
- The dictionary containing the origin input data of model.
-
- Returns
- -------
- gt_box3d_tensor : torch.Tensor
- The groundtruth bounding box tensor, shape (N, 8, 3).
- """
- gt_box3d_list = []
- # used to avoid repetitive bounding box
- object_id_list = []
-
- for cav_id, cav_content in data_dict.items():
- # used to project gt bounding box to ego space
- transformation_matrix = cav_content["transformation_matrix"]
-
- object_bbx_center = cav_content["object_bbx_center"]
- object_bbx_mask = cav_content["object_bbx_mask"]
- object_ids = cav_content["object_ids"]
- object_bbx_center = object_bbx_center[object_bbx_mask == 1]
-
- # convert center to corner
- object_bbx_corner = box_utils.boxes_to_corners_3d(object_bbx_center, self.params["order"])
- projected_object_bbx_corner = box_utils.project_box3d(object_bbx_corner.float(), transformation_matrix)
- gt_box3d_list.append(projected_object_bbx_corner)
-
- # append the corresponding ids
- object_id_list += object_ids
-
- # gt bbx 3d
- gt_box3d_list = torch.vstack(gt_box3d_list)
- # some of the bbx may be repetitive, use the id list to filter
- gt_box3d_selected_indices = [object_id_list.index(x) for x in set(object_id_list)]
- gt_box3d_tensor = gt_box3d_list[gt_box3d_selected_indices]
-
- # filter the gt_box to make sure all bbx are in the range
- mask = box_utils.get_mask_for_boxes_within_range_torch(gt_box3d_tensor)
- gt_box3d_tensor = gt_box3d_tensor[mask, :, :]
-
- return gt_box3d_tensor
-
- def generate_object_center(self, cav_contents, reference_lidar_pose):
- """
- Retrieve all objects in a format of (n, 7), where 7 represents
- x, y, z, l, w, h, yaw or x, y, z, h, w, l, yaw.
-
- Parameters
- ----------
- cav_contents : list
- List of dictionary, save all cavs' information.
-
- reference_lidar_pose : list
- The final target lidar pose with length 6.
-
- Returns
- -------
- object_np : np.ndarray
- Shape is (max_num, 7).
- mask : np.ndarray
- Shape is (max_num,).
- object_ids : list
- Length is number of bbx in current sample.
- """
-
- tmp_object_dict = {}
- for cav_content in cav_contents:
- tmp_object_dict.update(cav_content["params"]["vehicles"])
-
- output_dict = {}
- filter_range = self.params["anchor_args"]["cav_lidar_range"]
-
- box_utils.project_world_objects(tmp_object_dict, output_dict, reference_lidar_pose, filter_range, self.params["order"])
-
- object_np = np.zeros((self.params["max_num"], 7))
- mask = np.zeros(self.params["max_num"])
- object_ids = []
-
- for i, (object_id, object_bbx) in enumerate(output_dict.items()):
- object_np[i] = object_bbx[0, :]
- mask[i] = 1
- object_ids.append(object_id)
-
- return object_np, mask, object_ids
diff --git a/OpenCOOD/opencood/data_utils/post_processor/bev_postprocessor.py b/OpenCOOD/opencood/data_utils/post_processor/bev_postprocessor.py
deleted file mode 100644
index 057fae72..00000000
--- a/OpenCOOD/opencood/data_utils/post_processor/bev_postprocessor.py
+++ /dev/null
@@ -1,438 +0,0 @@
-"""
-Anchor-free 2d Generator
-"""
-
-import numpy as np
-import torch
-import torch.nn.functional as F
-
-from opencood.utils.transformation_utils import dist_to_continuous
-from opencood.data_utils.post_processor.base_postprocessor import BasePostprocessor
-from opencood.utils import box_utils
-from opencood.visualization import vis_utils
-
-
-class BevPostprocessor(BasePostprocessor):
- def __init__(self, anchor_params, train):
- super(BevPostprocessor, self).__init__(anchor_params, train)
- # self.geometry_param = anchor_params["geometry"]
- self.geometry_param = anchor_params["geometry_param"]
-
- # TODO
- # Hard coded for now. Need to calculate for our own training dataset
- self.target_mean = np.array([0.008, 0.001, 0.202, 0.2, 0.43, 1.368])
- self.target_std_dev = np.array([0.866, 0.5, 0.954, 0.668, 0.09, 0.111])
-
- def generate_anchor_box(self):
- return None
-
- def generate_label(self, **kwargs):
- """
- Generate targets for training.
-
- Parameters
- ----------
- kwargs : list
- gt_box_center:(max_num, 7)
-
- Returns
- -------
- label_dict : dict
- Dictionary that contains all target related info.
- """
-
- assert self.params["order"] == "lwh", "Currently BEV only support lwh bbx order."
- # (max_num, 7)
- gt_box_center = kwargs["gt_box_center"]
-
- # (max_num)
- masks = kwargs["mask"]
-
- # (n, 7)
- gt_box_center_valid = gt_box_center[masks == 1]
- # (n, 4, 3)
- bev_corners = box_utils.boxes_to_corners2d(gt_box_center_valid, self.params["order"])
-
- _ = gt_box_center_valid.shape[0] # n
- # (n, 4, 2)
- bev_corners = bev_corners[:, :, :2]
- yaw = gt_box_center_valid[:, -1]
- x, y = gt_box_center_valid[:, 0], gt_box_center_valid[:, 1]
- dx, dy = gt_box_center_valid[:, 3], gt_box_center_valid[:, 4]
- # (n, 6)
- reg_targets = np.column_stack([np.cos(yaw), np.sin(yaw), x, y, dx, dy])
-
- # target label map including classification and regression targets
- label_map = np.zeros(self.geometry_param["label_shape"])
- self.update_label_map(label_map, bev_corners, reg_targets)
- label_map = self.normalize_targets(label_map)
- label_dict = {
- # (7, label_shape[0], label_shape[1])
- "label_map": np.transpose(label_map, (2, 0, 1)).astype(np.float32),
- "bev_corners": bev_corners,
- }
- return label_dict
-
- def update_label_map(self, label_map, bev_corners, reg_targets):
- """
- Update label_map based on bbx and regression targets.
-
- Parameters
- ----------
- label_map : numpy.array
- Targets array for classification and regression tasks with
- the shape of label_shape.
-
- bev_corners : numpy.array
- The bbx corners in lidar frame with shape (n, 4, 2)
-
- reg_targets : numpy.array
- Array containing the regression targets information. It need to be
- further processed.
-
- """
- res = self.geometry_param["res"]
- downsample_rate = self.geometry_param["downsample_rate"]
-
- bev_origin = np.array([self.geometry_param["L1"], self.geometry_param["W1"]]).reshape(1, -1)
-
- # discretized bbx corner representations -- (n, 4, 2)
- bev_corners_dist = (bev_corners - bev_origin) / res / downsample_rate
- # generate the coordinates of m
- x = np.arange(self.geometry_param["label_shape"][0])
- y = np.arange(self.geometry_param["label_shape"][1])
- xx, yy = np.meshgrid(x, y)
-
- # (label_shape[0]*label_shape[1], 2)
- points = np.concatenate([xx.reshape(-1, 1), yy.reshape(-1, 1)], axis=-1)
- bev_origin_dist = bev_origin / res / downsample_rate
-
- # loop over each bbx, find the points within the bbx.
- for i in range(bev_corners.shape[0]):
- reg_target = reg_targets[i, :]
-
- # find discredited points in bbx
- points_in_box = box_utils.get_points_in_rotated_box(points, bev_corners_dist[i, ...])
- # convert points to continuous space
- points_continuous = dist_to_continuous(points_in_box, bev_origin_dist, res, downsample_rate)
- actual_reg_target = np.repeat(reg_target.reshape(1, -1), points_continuous.shape[0], axis=0)
- # build learning targets
- actual_reg_target[:, 2:4] = actual_reg_target[:, 2:4] - points_continuous
- actual_reg_target[:, 4:] = np.log(actual_reg_target[:, 4:])
-
- # update label map
- label_map[points_in_box[:, 0], points_in_box[:, 1], 0] = 1.0
- label_map[points_in_box[:, 0], points_in_box[:, 1], 1:] = actual_reg_target
-
- def normalize_targets(self, label_map):
- """
- Normalize label_map
-
- Parameters
- ----------
- label_map : numpy.array
- Targets array for classification and regression tasks with the
- shape of label_shape.
-
- Returns
- -------
- label_map: numpy.array
- Nromalized label_map.
-
- """
- label_map[..., 1:] = (label_map[..., 1:] - self.target_mean) / self.target_std_dev
- return label_map
-
- def denormalize_reg_map(self, reg_map):
- """
- Denormalize the regression map
-
- Parameters
- ----------
- reg_map : np.ndarray / torch.Tensor
- Regression output mapwith the shape of (label_shape[0],
- label_shape[1], 6).
-
- Returns
- -------
- reg_map : np.ndarray / torch.Tensor
- Denormalized regression map.
-
- """
- if isinstance(reg_map, np.ndarray):
- target_mean = self.target_mean
- target_std_dev = self.target_std_dev
-
- else:
- target_mean = torch.from_numpy(self.target_mean).to(reg_map.device)
- target_std_dev = torch.from_numpy(self.target_std_dev).to(reg_map.device)
- reg_map = reg_map * target_std_dev + target_mean
- return reg_map
-
- @staticmethod
- def collate_batch(label_batch_list):
- """
- Customized collate function for target label generation.
-
- Parameters
- ----------
- label_batch_list : list
- The list of dictionary that contains all labels for several
- frames.
-
- Returns
- -------
- processed_batch : dict
- Reformatted labels in torch tensor.
- """
- label_map_list = [x["label_map"][np.newaxis, ...] for x in label_batch_list]
- processed_batch = {
- # (batch_size, 7, label_shape[0], label_shape[1])
- "label_map": torch.from_numpy(np.concatenate(label_map_list, axis=0)),
- "bev_corners": [torch.from_numpy(x["bev_corners"]) for x in label_batch_list],
- }
- return processed_batch
-
- def decode_agent_predictions(self, cav_content, output):
- """
- Decode one agent's PIXOR output without projection or NMS.
-
- Parameters
- ----------
- cav_content : dict
- Local model input. It is accepted for a common postprocessor API.
- output : dict
- Raw output produced by the local detector.
-
- Returns
- -------
- tuple[torch.Tensor, torch.Tensor]
- Sender-frame 2D box corners and confidence scores.
- """
- del cav_content
- prob = torch.sigmoid(output["cls"].squeeze(0).squeeze(0))
- reg_map = output["reg"].squeeze(0).permute(1, 2, 0)
- reg_map = self.denormalize_reg_map(reg_map)
- mask = torch.gt(prob, self.params["target_args"]["score_threshold"])
- if mask.sum() == 0:
- return reg_map.new_empty((0, 4, 2)), prob.new_empty((0,))
- return self.reg_map_to_bbx_corners(reg_map, mask), prob[mask]
-
- def post_process_detections(self, boxes, scores):
- """
- Apply receiver-side NMS and range filtering to 2D detections.
-
- Parameters
- ----------
- boxes : torch.Tensor
- Ego-frame boxes with shape ``(N, 4, 2)``.
- scores : torch.Tensor
- Confidence scores with shape ``(N,)``.
-
- Returns
- -------
- tuple[torch.Tensor | None, torch.Tensor | None]
- Final boxes and scores, or two ``None`` values when empty.
- """
- if boxes.shape[0] == 0:
- return None, None
- keep_index = box_utils.nms_rotated(boxes, scores, self.params["nms_thresh"])
- boxes = boxes[keep_index]
- scores = scores[keep_index]
-
- range_mask = box_utils.get_mask_for_boxes_within_range_torch(boxes)
- boxes = boxes[range_mask]
- scores = scores[range_mask]
- assert scores.shape[0] == boxes.shape[0]
- return boxes, scores
-
- def post_process(self, data_dict, output_dict):
- """
- Process the outputs of the model to 2D bounding box.
- Step1: convert each cav's output to bounding box format
- Step2: project the bounding boxes to ego space.
- Step:3 NMS
-
- Parameters
- ----------
- data_dict : dict
- The dictionary containing the origin input data of model.
-
- output_dict :dict
- The dictionary containing the output of the model.
-
- Returns
- -------
- pred_box2d_tensor : torch.Tensor
- The prediction bounding box tensor after NMS.
-
- gt_box2d_tensor : torch.Tensor
- The groundtruth bounding box tensor.
- """
-
- # the final bounding box list
- pred_box2d_list = []
- pred_score_list = []
-
- for cav_id, cav_content in data_dict.items():
- assert cav_id in output_dict
- # the transformation matrix to ego space
- transformation_matrix = cav_content["transformation_matrix"]
-
- corners2d, scores = self.decode_agent_predictions(
- cav_content,
- output_dict[cav_id],
- )
- if corners2d.shape[0] > 0:
- # assume the z-diviation in transformation_matrix is small,
- # thus we can pad zeros to simulate the 3d transformation.
- # (number of high confidence bbx, 4, 3)
- box3d = F.pad(corners2d, (0, 1))
- # (number of high confidence bbx, 4, 2)
- projected_boxes2d = box_utils.project_points_by_matrix_torch(box3d.view(-1, 3), transformation_matrix)[:, :2]
-
- projected_boxes2d = projected_boxes2d.view(-1, 4, 2)
- pred_box2d_list.append(projected_boxes2d)
- pred_score_list.append(scores)
-
- if len(pred_box2d_list):
- pred_box2ds = torch.cat(pred_box2d_list, dim=0)
- pred_scores = torch.cat(pred_score_list, dim=0)
- else:
- return None, None
-
- return self.post_process_detections(pred_box2ds, pred_scores)
-
- def reg_map_to_bbx_corners(self, reg_map, mask):
- """
- Construct bbx from the regression output of the model.
-
- Parameters
- ----------
- reg_map : torch.Tensor
- Regression output of neural networks.
-
- mask : torch.Tensor
- Masks used to filter bbx.
-
- Returns
- -------
- corners : torch.Tensor
- Bbx output with shape (N, 4, 2).
-
- """
-
- assert len(reg_map.shape) == 3, "only support shape of label_shape i.e. (*, *, 6)"
- device = reg_map.device
-
- cos_t, sin_t, x, y, log_dx, log_dy = [tt.squeeze(-1) for tt in torch.chunk(reg_map, 6, dim=-1)]
- yaw = torch.atan2(sin_t, cos_t)
- dx, dy = log_dx.exp(), log_dy.exp()
-
- grid_size = self.geometry_param["res"] * self.geometry_param["downsample_rate"]
- grid_x = torch.arange(self.geometry_param["L1"], self.geometry_param["L2"], grid_size, dtype=torch.float32, device=device)
- grid_y = torch.arange(self.geometry_param["W1"], self.geometry_param["W2"], grid_size, dtype=torch.float32, device=device)
-
- xx, yy = torch.meshgrid([grid_x, grid_y])
- center_x = xx + x
- center_y = yy + y
-
- bbx2d = torch.stack([center_x, center_y, dx, dy, yaw], dim=-1)
- bbx2d = bbx2d[mask, :]
- corners = box_utils.boxes2d_to_corners2d(bbx2d)
-
- return corners
-
- def post_process_debug(self, data_dict, output_dict):
- """
- Process the outputs of the model to 2D bounding box for debug purpose.
- Step1: convert each cav's output to bounding box format
- Step2: project the bounding boxes to ego space.
- Step:3 NMS
-
- Parameters
- ----------
- data_dict : dict
- The dictionary containing the origin input data of model.
-
- output_dict :dict
- The dictionary containing the output of the model.
-
- Returns
- -------
- pred_box2d_tensor : torch.Tensor
- The prediction bounding box tensor after NMS.
- gt_box2d_tensor : torch.Tensor
- The groundtruth bounding box tensor.
- """
- # the final bounding box list
- pred_box2d_list = []
- pred_score_list = []
-
- # the transformation matrix to ego space
- transformation_matrix = data_dict["transformation_matrix"]
-
- # classification probability -- (label_shape[0], label_shape[1])
- prob = output_dict["cls"].squeeze(0).squeeze(0)
- prob = torch.sigmoid(prob)
-
- # regression map -- (label_shape[0], label_shape[1], 6)
- reg_map = output_dict["reg"].squeeze(0).permute(1, 2, 0)
- reg_map = self.denormalize_reg_map(reg_map)
-
- threshold = 0.5
- mask = torch.gt(prob, threshold)
-
- if mask.sum() > 0:
- # (number of high confidence bbx, 4, 2)
- corners2d = self.reg_map_to_bbx_corners(reg_map, mask)
- # assume the z-diviation in transformation_matrix is small,
- # thus we can pad zeros to simulate the 3d transformation.
- # (number of high confidence bbx, 4, 3)
- box3d = F.pad(corners2d, (0, 1))
-
- # (number of high confidence bbx, 4, 2)
- projected_boxes2d = box_utils.project_points_by_matrix_torch(box3d.view(-1, 3), transformation_matrix)[:, :2]
- projected_boxes2d = projected_boxes2d.view(-1, 4, 2)
- scores = prob[mask]
- pred_box2d_list.append(projected_boxes2d)
- pred_score_list.append(scores)
-
- pred_box2ds = torch.cat(pred_box2d_list, dim=0)
- pred_scores = torch.cat(pred_score_list, dim=0)
-
- keep_index = box_utils.nms_rotated(pred_box2ds, pred_scores, self.params["nms_thresh"])
- pred_box2ds = pred_box2ds[keep_index]
-
- # filter out the prediction out of the range.
- mask = box_utils.get_mask_for_boxes_within_range_torch(pred_box2ds)
- pred_box2ds = pred_box2ds[mask, :, :]
- return pred_box2ds
-
- @staticmethod
- def visualize(pred_box_tensor, gt_tensor, pcd, show_vis, save_path, dataset=None):
- """
- Visualize the BEV 2D prediction, ground truth with point cloud together.
-
- Parameters
- ----------
- pred_box_tensor : torch.Tensor
- (N, 8, 3) prediction.
-
- gt_tensor : torch.Tensor
- (N, 8, 3) groundtruth bbx
-
- pcd : torch.Tensor
- PointCloud, (N, 4).
-
- show_vis : bool
- Whether to show visualization.
-
- save_path : str
- Save the visualization results to given path.
-
- dataset : BaseDataset
- opencood dataset object.
- """
- assert dataset is not None, "dataset argument can't be None"
- vis_utils.visualize_single_sample_output_bev(pred_box_tensor, gt_tensor, pcd, dataset, show_vis, save_path)
diff --git a/OpenCOOD/opencood/data_utils/post_processor/ciassd_postprocessor.py b/OpenCOOD/opencood/data_utils/post_processor/ciassd_postprocessor.py
deleted file mode 100644
index 59487c31..00000000
--- a/OpenCOOD/opencood/data_utils/post_processor/ciassd_postprocessor.py
+++ /dev/null
@@ -1,200 +0,0 @@
-"""
-3D Anchor Generator for Voxel
-"""
-
-import numpy as np
-import torch
-
-from opencood.data_utils.post_processor.voxel_postprocessor import VoxelPostprocessor
-from opencood.utils import box_utils
-
-
-class CiassdPostprocessor(VoxelPostprocessor):
- def __init__(self, anchor_params, train):
- super(CiassdPostprocessor, self).__init__(anchor_params, train)
- self.train = train
- self.anchor_num = self.params["anchor_args"]["num"]
-
- def decode_agent_predictions(self, cav_content, output):
- """
- Decode one agent's CIA-SSD output without projection or NMS.
-
- Parameters
- ----------
- cav_content : dict
- Local model input containing the configured anchors.
- output : dict
- Raw output produced by CIA-SSD.
-
- Returns
- -------
- tuple[torch.Tensor, torch.Tensor]
- Sender-frame box corners and confidence scores.
- """
- preds_dict = output["preds_dict_stage1"]
- prob = torch.sigmoid(preds_dict["cls_preds"].permute(0, 2, 3, 1).contiguous())
- reg = preds_dict["box_preds"].permute(0, 2, 3, 1).contiguous()
- iou = preds_dict["iou_preds"].permute(0, 2, 3, 1).contiguous().reshape(1, -1)
- direction = preds_dict["dir_cls_preds"].permute(0, 2, 3, 1).contiguous().reshape(1, -1, 2)
-
- batch_box3d = self.delta_to_boxes3d(reg, cav_content["anchor_box"], False)
- mask = torch.gt(prob, self.params["target_args"]["score_threshold"]).view(1, -1)
- mask_reg = mask.unsqueeze(2).repeat(1, 1, 7)
- assert batch_box3d.shape[0] == 1
- boxes3d = torch.masked_select(batch_box3d[0], mask_reg[0]).view(-1, 7)
- scores = torch.masked_select(prob.reshape(-1), mask[0])
- if boxes3d.shape[0] == 0:
- return boxes3d.new_empty((0, 8, 3)), scores
-
- direction_labels = torch.max(direction, dim=-1)[1][mask]
- adjusted_iou = (iou + 1) * 0.5
- scores = scores * torch.pow(adjusted_iou.masked_select(mask), 4)
- opposite_direction = (boxes3d[..., -1] > 0) ^ (direction_labels.byte() == 1)
- boxes3d[..., -1] += torch.where(
- opposite_direction,
- torch.tensor(np.pi).type_as(boxes3d),
- torch.tensor(0.0).type_as(boxes3d),
- )
- return box_utils.boxes_to_corners_3d(
- boxes3d,
- order=self.params["order"],
- ), scores
-
- def post_process(self, data_dict, output_dict):
- """
- Process the outputs of the model to 2D/3D bounding box.
- Step1: convert each cav's output to bounding box format
- Step2: project the bounding boxes to ego space.
- Step:3 NMS
-
- Parameters
- ----------
- data_dict : dict
- The dictionary containing the origin input data of model.
-
- output_dict :dict
- The dictionary containing the output of the model.
-
- Returns
- -------
- pred_box3d_tensor : torch.Tensor
- The prediction bounding box tensor after NMS.
- gt_box3d_tensor : torch.Tensor
- The groundtruth bounding box tensor.
- """
- # the final bounding box list
- global batch_num_box_count
- pred_box3d_original_list = []
- pred_box3d_list = []
- pred_box2d_list = []
-
- for cav_id, cav_content in data_dict.items():
- assert cav_id in output_dict
- # the transformation matrix to ego space
- if "transformation_matrix" in cav_content:
- transformation_matrix = cav_content["transformation_matrix"]
- else:
- transformation_matrix = torch.from_numpy(np.identity(4)).float().to(cav_content["anchor_box"].device)
-
- # (H, W, anchor_num, 7)
- anchor_box = cav_content["anchor_box"]
-
- # prediction result
- preds_dict = output_dict[cav_id]["preds_dict_stage1"]
-
- # preds
- prob = preds_dict["cls_preds"]
- prob = torch.sigmoid(prob.permute(0, 2, 3, 1).contiguous())
- reg = preds_dict["box_preds"].permute(0, 2, 3, 1).contiguous()
- iou = preds_dict["iou_preds"].permute(0, 2, 3, 1).contiguous().reshape(1, -1)
- dir = preds_dict["dir_cls_preds"].permute(0, 2, 3, 1).contiguous().reshape(1, -1, 2)
-
- # convert regression map back to bounding box
- # (N, W*L*anchor_num, 7)
- batch_box3d = self.delta_to_boxes3d(reg, anchor_box, False)
- mask = torch.gt(prob, self.params["target_args"]["score_threshold"])
- batch_num_box_count = [int(m.sum()) for m in mask]
- mask = mask.view(1, -1)
- mask_reg = mask.unsqueeze(2).repeat(1, 1, 7)
-
- # during validation/testing, the batch size should be 1
- if not self.train:
- assert batch_box3d.shape[0] == 1
-
- boxes3d = torch.masked_select(batch_box3d.view(-1, 7), mask_reg[0]).view(-1, 7)
- scores = torch.masked_select(prob.view(-1), mask[0])
-
- dir_labels = torch.max(dir, dim=-1)[1]
- dir_labels = dir_labels[mask]
- # top_labels = torch.zeros([scores.shape[0]], dtype=torch.long).cuda()
- if scores.shape[0] != 0:
- iou = (iou + 1) * 0.5
- scores = scores * torch.pow(iou.masked_select(mask), 4)
- # correct_direction
- top_labels = (boxes3d[..., -1] > 0) ^ (dir_labels.byte() == 1)
- boxes3d[..., -1] += torch.where(top_labels, torch.tensor(np.pi).type_as(boxes3d), torch.tensor(0.0).type_as(boxes3d))
- pred_box3d_original_list.append(boxes3d.detach())
-
- # convert output to bounding box
- if len(boxes3d) != 0:
- # (N, 8, 3)
- boxes3d_corner = box_utils.boxes_to_corners_3d(boxes3d, order=self.params["order"])
- # (N, 8, 3)
- projected_boxes3d = box_utils.project_box3d(boxes3d_corner, transformation_matrix)
- # convert 3d bbx to 2d, (N,4)
- projected_boxes2d = box_utils.corner_to_standup_box_torch(projected_boxes3d)
- # (N, 5)
- boxes2d_score = torch.cat((projected_boxes2d, scores.unsqueeze(1)), dim=1)
-
- pred_box2d_list.append(boxes2d_score)
- pred_box3d_list.append(projected_boxes3d)
-
- if len(pred_box2d_list) == 0 or len(pred_box3d_list) == 0:
- return None, None
- # shape: (N, 5)
- pred_box2d_list = torch.vstack(pred_box2d_list)
- # scores
- scores = pred_box2d_list[:, -1]
- # predicted 3d bbx
- pred_box3d_tensor = torch.vstack(pred_box3d_list)
- pred_box3d_original = torch.vstack(pred_box3d_original_list)
-
- if not self.train:
- # remove large bbx
- keep_index_1 = box_utils.remove_large_pred_bbx(pred_box3d_tensor)
- keep_index_2 = box_utils.remove_bbx_abnormal_z(pred_box3d_tensor)
- keep_index = torch.logical_and(keep_index_1, keep_index_2)
-
- pred_box3d_tensor = pred_box3d_tensor[keep_index]
- scores = scores[keep_index]
-
- # nms
- keep_index = box_utils.nms_rotated(pred_box3d_tensor, scores, self.params["nms_thresh"])
-
- pred_box3d_tensor = pred_box3d_tensor[keep_index]
-
- # select cooresponding score
- scores = scores[keep_index]
-
- # filter out the prediction out of the range.
- mask = box_utils.get_mask_for_boxes_within_range_torch(pred_box3d_tensor)
- pred_box3d_tensor = pred_box3d_tensor[mask, :, :]
- scores = scores[mask]
-
- assert scores.shape[0] == pred_box3d_tensor.shape[0]
- return pred_box3d_tensor, scores
- else:
- cur_idx = 0
- batch_pred_boxes3d = []
- batch_scores = []
- for n in batch_num_box_count:
- cur_boxes = pred_box3d_tensor[cur_idx : cur_idx + n]
- cur_scores = scores[cur_idx : cur_idx + n]
- # nms
- keep_index = box_utils.nms_rotated(cur_boxes, cur_scores, self.params["nms_thresh"])
- cur_boxes = pred_box3d_original[cur_idx : cur_idx + n] # [:, [0, 1, 2, 5, 4, 3, 6]] # hwl -> lwh
- batch_pred_boxes3d.append(cur_boxes[keep_index])
- batch_scores.append(cur_scores[keep_index])
- cur_idx += n
-
- return batch_pred_boxes3d, batch_scores
diff --git a/OpenCOOD/opencood/data_utils/post_processor/fpvrcnn_postprocessor.py b/OpenCOOD/opencood/data_utils/post_processor/fpvrcnn_postprocessor.py
deleted file mode 100644
index c95d259d..00000000
--- a/OpenCOOD/opencood/data_utils/post_processor/fpvrcnn_postprocessor.py
+++ /dev/null
@@ -1,196 +0,0 @@
-"""
-3D Anchor Generator for Voxel
-"""
-
-import numpy as np
-import torch
-
-from opencood.data_utils.post_processor.voxel_postprocessor import VoxelPostprocessor
-from opencood.utils import box_utils
-from opencood.utils import common_utils
-
-
-class FpvrcnnPostprocessor(VoxelPostprocessor):
- def __init__(self, anchor_params, train):
- super(FpvrcnnPostprocessor, self).__init__(anchor_params, train)
-
- def post_process(self, data_dict, output_dict, stage1=False):
- if stage1:
- return self.post_process_stage1(data_dict, output_dict)
- else:
- return self.post_process_stage2(data_dict)
-
- def post_process_stage1(self, data_dict, output_dict):
- """
- Process the outputs of the model to 2D/3D bounding box.
- Step1: convert each cav's output to bounding box format
- Step2: project the bounding boxes to ego space.
- Step:3 NMS
-
- Parameters
- ----------
- data_dict : dict
- The dictionary containing the origin input data of model.
-
- output_dict :dict
- The dictionary containing the output of the model.
-
- Returns
- -------
- pred_box3d_tensor : torch.Tensor
- The prediction bounding box tensor after NMS.
- gt_box3d_tensor : torch.Tensor
- The groundtruth bounding box tensor.
- """
- # the final bounding box list
- pred_box3d_original_list = []
- pred_box3d_list = []
- pred_box2d_list = []
-
- for cav_id, cav_content in data_dict.items():
- assert cav_id in output_dict
- # the transformation matrix to ego space
- if "transformation_matrix" in cav_content:
- transformation_matrix = cav_content["transformation_matrix"]
- else:
- transformation_matrix = torch.from_numpy(np.identity(4)).float().to(cav_content["anchor_box"].device)
-
- # (H, W, anchor_num, 7)
- anchor_box = cav_content["anchor_box"]
-
- # prediction result
- preds_dict = output_dict[cav_id]["preds_dict_stage1"]
-
- # preds
- prob = preds_dict["cls_preds"]
- prob = torch.sigmoid(prob.permute(0, 2, 3, 1).contiguous())
- reg = preds_dict["box_preds"].permute(0, 2, 3, 1).contiguous()
- iou = torch.sigmoid(preds_dict["iou_preds"].permute(0, 2, 3, 1).contiguous()).reshape(1, -1)
- dir = preds_dict["dir_cls_preds"].permute(0, 2, 3, 1).contiguous().reshape(1, -1, 2)
-
- # convert regression map back to bounding box
- # (N, W*L*anchor_num, 7)
- batch_box3d = self.delta_to_boxes3d(reg, anchor_box, False)
- mask = torch.gt(prob, self.params["target_args"]["score_threshold"])
- batch_num_box_count = [int(m.sum()) for m in mask]
- mask = mask.view(1, -1)
- mask_reg = mask.unsqueeze(2).repeat(1, 1, 7)
-
- # during validation/testing, the batch size should be 1
- if not self.train:
- assert batch_box3d.shape[0] == 1
-
- boxes3d = torch.masked_select(batch_box3d.view(-1, 7), mask_reg[0]).view(-1, 7) # hwl
- scores = torch.masked_select(prob.view(-1), mask[0])
-
- dir_labels = torch.max(dir, dim=-1)[1]
- dir_labels = dir_labels[mask]
- # top_labels = torch.zeros([scores.shape[0]], dtype=torch.long).cuda()
- if scores.shape[0] != 0:
- iou = torch.clamp(iou, min=0.0, max=1.0)
- iou = (iou + 1) * 0.5
- scores = scores * torch.pow(iou.masked_select(mask), 4)
- # correct_direction
- top_labels = (boxes3d[..., -1] > 0) ^ (dir_labels.byte() == 1)
- boxes3d[..., -1] += torch.where(top_labels, torch.tensor(np.pi).type_as(boxes3d), torch.tensor(0.0).type_as(boxes3d))
-
- # filter invalid boxes
- keep_idx = torch.logical_and((boxes3d[:, 3:6] > 1).all(dim=1), (boxes3d[:, 3:6] < 10).all(dim=1))
- idx_start = 0
- count = []
- for i, n in enumerate(batch_num_box_count):
- count.append(int(keep_idx[idx_start : idx_start + n].sum()))
- batch_num_box_count = count
- boxes3d = boxes3d[keep_idx]
- scores = scores[keep_idx]
-
- # if the number of boxes is too huge, this would consume a lot of memory in the second stage
- # therefore, randomly select some boxes if the box number is too big at the beginning of the training
- if len(boxes3d) > 300:
- keep_idx = torch.multinomial(scores, 300)
- idx_start = 0
- count = []
- for i, n in enumerate(batch_num_box_count):
- count.append(int(torch.logical_and(keep_idx >= idx_start, keep_idx < idx_start + n).sum()))
- batch_num_box_count = count
- boxes3d = boxes3d[keep_idx]
- scores = scores[keep_idx]
-
- pred_box3d_original_list.append(boxes3d.detach())
-
- # convert output to bounding box
- if len(boxes3d) != 0:
- # (N, 8, 3)
- boxes3d_corner = box_utils.boxes_to_corners_3d(boxes3d, order=self.params["order"])
- # (N, 8, 3)
- projected_boxes3d = box_utils.project_box3d(boxes3d_corner, transformation_matrix)
- # convert 3d bbx to 2d, (N,4)
- projected_boxes2d = box_utils.corner_to_standup_box_torch(projected_boxes3d)
- # (N, 5)
- boxes2d_score = torch.cat((projected_boxes2d, scores.unsqueeze(1)), dim=1)
-
- pred_box2d_list.append(boxes2d_score)
- pred_box3d_list.append(projected_boxes3d)
-
- if len(pred_box2d_list) == 0 or len(pred_box3d_list) == 0:
- return None, None
- # shape: (N, 5)
- pred_box2d_list = torch.vstack(pred_box2d_list)
- # scores
- scores = pred_box2d_list[:, -1]
- # predicted 3d bbx
- pred_box3d_tensor = torch.vstack(pred_box3d_list)
- pred_box3d_original = torch.vstack(pred_box3d_original_list)
-
- cur_idx = 0
- batch_pred_boxes3d = []
- batch_scores = []
- for n in batch_num_box_count:
- cur_boxes = pred_box3d_tensor[cur_idx : cur_idx + n]
- cur_scores = scores[cur_idx : cur_idx + n]
- # nms
- keep_index = box_utils.nms_rotated(cur_boxes, cur_scores, self.params["nms_thresh"])
- cur_boxes = pred_box3d_original[cur_idx : cur_idx + n][:, [0, 1, 2, 5, 4, 3, 6]] # hwl -> lwh
- batch_pred_boxes3d.append(cur_boxes[keep_index])
- batch_scores.append(cur_scores[keep_index])
- cur_idx += n
-
- return batch_pred_boxes3d, batch_scores
-
- def post_process_stage2(self, data_dict):
- from opencood.pcdet_utils.iou3d_nms.iou3d_nms_utils import nms_gpu
-
- output_dict = data_dict["ego"]["fpvrcnn_out"]
- if output_dict is None:
- return None, None
- label_dict = data_dict["ego"]["rcnn_label_dict"]
- rcnn_cls = output_dict["rcnn_cls"].sigmoid().view(-1)
- rcnn_iou = output_dict["rcnn_iou"].view(-1)
- rcnn_score = rcnn_cls * rcnn_iou**4
- rcnn_reg = output_dict["rcnn_reg"].view(-1, 7)
- rois_anchor = label_dict["rois_anchor"]
- rois = label_dict["rois"]
- roi_center = rois[:, 0:3]
- roi_ry = rois[:, 6] % (2 * np.pi)
-
- boxes_local = box_utils.box_decode(rcnn_reg, rois_anchor)
- # boxes_local = rcnn_reg + rois_anchor
- detections = common_utils.rotate_points_along_z(points=boxes_local.view(-1, 1, boxes_local.shape[-1]), angle=roi_ry.view(-1)).view(
- -1, boxes_local.shape[-1]
- )
- detections[:, :3] = detections[:, :3] + roi_center
- detections[:, 6] = detections[:, 6] + roi_ry
- mask = rcnn_score >= 0.01
- detections = detections[mask]
- scores = rcnn_score[mask]
- # gt_boxes = label_dict['gt_of_rois_src'][mask]
- mask = nms_gpu(detections, scores, thresh=0.01)[0]
- boxes3d = detections[mask][:, [0, 1, 2, 5, 4, 3, 6]] # lwh -> hwl
- projected_boxes3d = None
- if len(boxes3d) != 0:
- # (N, 8, 3)
- boxes3d_corner = box_utils.boxes_to_corners_3d(boxes3d, order=self.params["order"])
- # (N, 8, 3)
- projected_boxes3d = box_utils.project_box3d(boxes3d_corner, data_dict["ego"]["transformation_matrix"])
-
- return projected_boxes3d, scores[mask]
diff --git a/OpenCOOD/opencood/data_utils/post_processor/voxel_postprocessor.py b/OpenCOOD/opencood/data_utils/post_processor/voxel_postprocessor.py
deleted file mode 100644
index beadd119..00000000
--- a/OpenCOOD/opencood/data_utils/post_processor/voxel_postprocessor.py
+++ /dev/null
@@ -1,391 +0,0 @@
-"""
-3D Anchor Generator for Voxel
-"""
-
-import math
-import sys
-
-import numpy as np
-import torch
-
-from opencood.data_utils.post_processor.base_postprocessor import BasePostprocessor
-from opencood.utils import box_utils
-from opencood.utils.box_overlaps import bbox_overlaps
-from opencood.visualization import vis_utils
-
-
-class VoxelPostprocessor(BasePostprocessor):
- def __init__(self, anchor_params, train):
- super(VoxelPostprocessor, self).__init__(anchor_params, train)
- self.anchor_num = self.params["anchor_args"]["num"]
-
- def generate_anchor_box(self):
- W = self.params["anchor_args"]["W"]
- H = self.params["anchor_args"]["H"]
-
- length = self.params["anchor_args"]["l"]
- w = self.params["anchor_args"]["w"]
- h = self.params["anchor_args"]["h"]
- r = self.params["anchor_args"]["r"]
-
- assert self.anchor_num == len(r)
- r = [math.radians(ele) for ele in r]
-
- vh = self.params["anchor_args"]["vh"]
- vw = self.params["anchor_args"]["vw"]
-
- xrange = [self.params["anchor_args"]["cav_lidar_range"][0], self.params["anchor_args"]["cav_lidar_range"][3]]
- yrange = [self.params["anchor_args"]["cav_lidar_range"][1], self.params["anchor_args"]["cav_lidar_range"][4]]
-
- if "feature_stride" in self.params["anchor_args"]:
- feature_stride = self.params["anchor_args"]["feature_stride"]
- else:
- feature_stride = 2
-
- x = np.linspace(xrange[0] + vw, xrange[1] - vw, W // feature_stride)
- y = np.linspace(yrange[0] + vh, yrange[1] - vh, H // feature_stride)
-
- cx, cy = np.meshgrid(x, y)
- cx = np.tile(cx[..., np.newaxis], self.anchor_num)
- cy = np.tile(cy[..., np.newaxis], self.anchor_num)
- cz = np.ones_like(cx) * -1.0
-
- w = np.ones_like(cx) * w
- length = np.ones_like(cx) * length
- h = np.ones_like(cx) * h
-
- r_ = np.ones_like(cx)
- for i in range(self.anchor_num):
- r_[..., i] = r[i]
-
- if self.params["order"] == "hwl":
- anchors = np.stack([cx, cy, cz, h, w, length, r_], axis=-1)
- elif self.params["order"] == "lhw":
- anchors = np.stack([cx, cy, cz, length, h, w, r_], axis=-1)
- else:
- sys.exit("Unknown bbx order.")
-
- return anchors
-
- def generate_label(self, **kwargs):
- """
- Generate targets for training.
-
- Parameters
- ----------
- argv : list
- gt_box_center:(max_num, 7), anchor:(H, W, anchor_num, 7)
-
- Returns
- -------
- label_dict : dict
- Dictionary that contains all target related info.
- """
- assert self.params["order"] == "hwl", "Currently Voxel only supporthwl bbx order."
- # (max_num, 7)
- gt_box_center = kwargs["gt_box_center"]
- # (H, W, anchor_num, 7)
- anchors = kwargs["anchors"]
- # (max_num)
- masks = kwargs["mask"]
-
- # (H, W)
- feature_map_shape = anchors.shape[:2]
-
- # (H*W*anchor_num, 7)
- anchors = anchors.reshape(-1, 7)
- # normalization factor, (H * W * anchor_num)
- anchors_d = np.sqrt(anchors[:, 4] ** 2 + anchors[:, 5] ** 2)
-
- # (H, W, 2)
- pos_equal_one = np.zeros((*feature_map_shape, self.anchor_num))
- neg_equal_one = np.zeros((*feature_map_shape, self.anchor_num))
- # (H, W, self.anchor_num * 7)
- targets = np.zeros((*feature_map_shape, self.anchor_num * 7))
-
- # (n, 7)
- gt_box_center_valid = gt_box_center[masks == 1]
- # (n, 8, 3)
- gt_box_corner_valid = box_utils.boxes_to_corners_3d(gt_box_center_valid, self.params["order"])
- # (H*W*anchor_num, 8, 3)
- anchors_corner = box_utils.boxes_to_corners_3d(anchors, order=self.params["order"])
- # (H*W*anchor_num, 4)
- anchors_standup_2d = box_utils.corner2d_to_standup_box(anchors_corner)
- # (n, 4)
- gt_standup_2d = box_utils.corner2d_to_standup_box(gt_box_corner_valid)
-
- # (H*W*anchor_n)
- iou = bbox_overlaps(
- np.ascontiguousarray(anchors_standup_2d).astype(np.float32),
- np.ascontiguousarray(gt_standup_2d).astype(np.float32),
- )
-
- # the anchor boxes has the largest iou across
- # shape: (n)
- id_highest = np.argmax(iou.T, axis=1)
- # [0, 1, 2, ..., n-1]
- id_highest_gt = np.arange(iou.T.shape[0])
- # make sure all highest iou is larger than 0
- mask = iou.T[id_highest_gt, id_highest] > 0
- id_highest, id_highest_gt = id_highest[mask], id_highest_gt[mask]
-
- # find anchors iou > params['pos_iou']
- id_pos, id_pos_gt = np.where(iou > self.params["target_args"]["pos_threshold"])
- # find anchors iou < params['neg_iou']
- id_neg = np.where(np.sum(iou < self.params["target_args"]["neg_threshold"], axis=1) == iou.shape[1])[0]
- id_pos = np.concatenate([id_pos, id_highest])
- id_pos_gt = np.concatenate([id_pos_gt, id_highest_gt])
- id_pos, index = np.unique(id_pos, return_index=True)
- id_pos_gt = id_pos_gt[index]
- id_neg.sort()
-
- # cal the target and set the equal one
- index_x, index_y, index_z = np.unravel_index(id_pos, (*feature_map_shape, self.anchor_num))
- pos_equal_one[index_x, index_y, index_z] = 1
-
- # calculate the targets
- targets[index_x, index_y, np.array(index_z) * 7] = (gt_box_center[id_pos_gt, 0] - anchors[id_pos, 0]) / anchors_d[id_pos]
- targets[index_x, index_y, np.array(index_z) * 7 + 1] = (gt_box_center[id_pos_gt, 1] - anchors[id_pos, 1]) / anchors_d[id_pos]
- targets[index_x, index_y, np.array(index_z) * 7 + 2] = (gt_box_center[id_pos_gt, 2] - anchors[id_pos, 2]) / anchors[id_pos, 3]
- targets[index_x, index_y, np.array(index_z) * 7 + 3] = np.log(gt_box_center[id_pos_gt, 3] / anchors[id_pos, 3])
- targets[index_x, index_y, np.array(index_z) * 7 + 4] = np.log(gt_box_center[id_pos_gt, 4] / anchors[id_pos, 4])
- targets[index_x, index_y, np.array(index_z) * 7 + 5] = np.log(gt_box_center[id_pos_gt, 5] / anchors[id_pos, 5])
- targets[index_x, index_y, np.array(index_z) * 7 + 6] = gt_box_center[id_pos_gt, 6] - anchors[id_pos, 6]
-
- index_x, index_y, index_z = np.unravel_index(id_neg, (*feature_map_shape, self.anchor_num))
- neg_equal_one[index_x, index_y, index_z] = 1
-
- # to avoid a box be pos/neg in the same time
- index_x, index_y, index_z = np.unravel_index(id_highest, (*feature_map_shape, self.anchor_num))
- neg_equal_one[index_x, index_y, index_z] = 0
-
- label_dict = {"pos_equal_one": pos_equal_one, "neg_equal_one": neg_equal_one, "targets": targets}
-
- return label_dict
-
- @staticmethod
- def collate_batch(label_batch_list):
- """
- Customized collate function for target label generation.
-
- Parameters
- ----------
- label_batch_list : list
- The list of dictionary that contains all labels for several
- frames.
-
- Returns
- -------
- target_batch : dict
- Reformatted labels in torch tensor.
- """
- pos_equal_one = []
- neg_equal_one = []
- targets = []
-
- for i in range(len(label_batch_list)):
- pos_equal_one.append(label_batch_list[i]["pos_equal_one"])
- neg_equal_one.append(label_batch_list[i]["neg_equal_one"])
- targets.append(label_batch_list[i]["targets"])
-
- pos_equal_one = torch.from_numpy(np.array(pos_equal_one))
- neg_equal_one = torch.from_numpy(np.array(neg_equal_one))
- targets = torch.from_numpy(np.array(targets))
-
- return {"targets": targets, "pos_equal_one": pos_equal_one, "neg_equal_one": neg_equal_one}
-
- def decode_agent_predictions(self, cav_content, output):
- """
- Decode one agent's detector output without projection or NMS.
-
- Parameters
- ----------
- cav_content : dict
- Local model input containing the configured anchors.
- output : dict
- Raw output produced by the local detector.
-
- Returns
- -------
- tuple[torch.Tensor, torch.Tensor]
- Sender-frame box corners and confidence scores.
- """
- anchor_box = cav_content["anchor_box"]
- prob = torch.sigmoid(output["psm"].permute(0, 2, 3, 1)).reshape(1, -1)
- batch_box3d = self.delta_to_boxes3d(output["rm"], anchor_box)
- mask = torch.gt(prob, self.params["target_args"]["score_threshold"]).view(1, -1)
- mask_reg = mask.unsqueeze(2).repeat(1, 1, 7)
-
- assert batch_box3d.shape[0] == 1
- boxes3d = torch.masked_select(batch_box3d[0], mask_reg[0]).view(-1, 7)
- scores = torch.masked_select(prob[0], mask[0])
- if boxes3d.shape[0] == 0:
- return boxes3d.new_empty((0, 8, 3)), scores
- return box_utils.boxes_to_corners_3d(boxes3d, order=self.params["order"]), scores
-
- def post_process_detections(self, boxes, scores):
- """
- Apply receiver-side filtering and NMS to decoded 3D detections.
-
- Parameters
- ----------
- boxes : torch.Tensor
- Ego-frame boxes with shape ``(N, 8, 3)``.
- scores : torch.Tensor
- Confidence scores with shape ``(N,)``.
-
- Returns
- -------
- tuple[torch.Tensor | None, torch.Tensor | None]
- Final boxes and scores, or two ``None`` values when empty.
- """
- if boxes.shape[0] == 0:
- return None, None
-
- keep_index = torch.logical_and(
- box_utils.remove_large_pred_bbx(boxes),
- box_utils.remove_bbx_abnormal_z(boxes),
- )
- boxes = boxes[keep_index]
- scores = scores[keep_index]
-
- keep_index = box_utils.nms_rotated(boxes, scores, self.params["nms_thresh"])
- boxes = boxes[keep_index]
- scores = scores[keep_index]
-
- range_mask = box_utils.get_mask_for_boxes_within_range_torch(boxes)
- boxes = boxes[range_mask]
- scores = scores[range_mask]
- assert scores.shape[0] == boxes.shape[0]
- return boxes, scores
-
- def post_process(self, data_dict, output_dict):
- """
- Process the outputs of the model to 2D/3D bounding box.
- Step1: convert each cav's output to bounding box format
- Step2: project the bounding boxes to ego space.
- Step:3 NMS
-
- Parameters
- ----------
- data_dict : dict
- The dictionary containing the origin input data of model.
-
- output_dict :dict
- The dictionary containing the output of the model.
-
- Returns
- -------
- pred_box3d_tensor : torch.Tensor
- The prediction bounding box tensor after NMS.
- gt_box3d_tensor : torch.Tensor
- The groundtruth bounding box tensor.
- """
- # the final bounding box list
- pred_box3d_list = []
- pred_box2d_list = []
-
- for cav_id, cav_content in data_dict.items():
- assert cav_id in output_dict
- # the transformation matrix to ego space
- transformation_matrix = cav_content["transformation_matrix"]
-
- boxes3d_corner, scores = self.decode_agent_predictions(
- cav_content,
- output_dict[cav_id],
- )
- if len(boxes3d_corner) != 0:
- # (N, 8, 3)
- projected_boxes3d = box_utils.project_box3d(boxes3d_corner, transformation_matrix)
- # convert 3d bbx to 2d, (N,4)
- projected_boxes2d = box_utils.corner_to_standup_box_torch(projected_boxes3d)
- # (N, 5)
- boxes2d_score = torch.cat((projected_boxes2d, scores.unsqueeze(1)), dim=1)
-
- pred_box2d_list.append(boxes2d_score)
- pred_box3d_list.append(projected_boxes3d)
-
- if len(pred_box2d_list) == 0 or len(pred_box3d_list) == 0:
- return None, None
- pred_box2d_tensor = torch.vstack(pred_box2d_list)
- return self.post_process_detections(
- torch.vstack(pred_box3d_list),
- pred_box2d_tensor[:, -1],
- )
-
- @staticmethod
- def delta_to_boxes3d(deltas, anchors, channel_swap=True):
- """
- Convert the output delta to 3d bbx.
-
- Parameters
- ----------
- deltas : torch.Tensor
- (N, W, L, 14)
- anchors : torch.Tensor
- (W, L, 2, 7) -> xyzhwlr
- channel_swap : bool
- Whether to swap the channel of deltas. It is only false when using
- FPV-RCNN
-
- Returns
- -------
- box3d : torch.Tensor
- (N, W*L*2, 7)
- """
- # batch size
- N = deltas.shape[0]
- if channel_swap:
- deltas = deltas.permute(0, 2, 3, 1).contiguous().view(N, -1, 7)
- else:
- deltas = deltas.contiguous().view(N, -1, 7)
-
- boxes3d = torch.zeros_like(deltas)
- if deltas.is_cuda:
- anchors = anchors.cuda()
- boxes3d = boxes3d.cuda()
-
- # (W*L*2, 7)
- anchors_reshaped = anchors.view(-1, 7).float()
- # the diagonal of the anchor 2d box, (W*L*2)
- anchors_d = torch.sqrt(anchors_reshaped[:, 4] ** 2 + anchors_reshaped[:, 5] ** 2)
- anchors_d = anchors_d.repeat(N, 2, 1).transpose(1, 2)
- anchors_reshaped = anchors_reshaped.repeat(N, 1, 1)
-
- # Inv-normalize to get xyz
- boxes3d[..., [0, 1]] = torch.mul(deltas[..., [0, 1]], anchors_d) + anchors_reshaped[..., [0, 1]]
- boxes3d[..., [2]] = torch.mul(deltas[..., [2]], anchors_reshaped[..., [3]]) + anchors_reshaped[..., [2]]
- # hwl
- boxes3d[..., [3, 4, 5]] = torch.exp(deltas[..., [3, 4, 5]]) * anchors_reshaped[..., [3, 4, 5]]
- # yaw angle
- boxes3d[..., 6] = deltas[..., 6] + anchors_reshaped[..., 6]
-
- return boxes3d
-
- @staticmethod
- def visualize(pred_box_tensor, gt_tensor, pcd, show_vis, save_path, dataset=None):
- """
- Visualize the prediction, ground truth with point cloud together.
-
- Parameters
- ----------
- pred_box_tensor : torch.Tensor
- (N, 8, 3) prediction.
-
- gt_tensor : torch.Tensor
- (N, 8, 3) groundtruth bbx
-
- pcd : torch.Tensor
- PointCloud, (N, 4).
-
- show_vis : bool
- Whether to show visualization.
-
- save_path : str
- Save the visualization results to given path.
-
- dataset : BaseDataset
- opencood dataset object.
-
- """
- vis_utils.visualize_single_sample_output_gt(pred_box_tensor, gt_tensor, pcd, show_vis, save_path)
diff --git a/OpenCOOD/opencood/data_utils/pre_processor/__init__.py b/OpenCOOD/opencood/data_utils/pre_processor/__init__.py
deleted file mode 100644
index cb7304fb..00000000
--- a/OpenCOOD/opencood/data_utils/pre_processor/__init__.py
+++ /dev/null
@@ -1,21 +0,0 @@
-from opencood.data_utils.pre_processor.base_preprocessor import BasePreprocessor
-from opencood.data_utils.pre_processor.voxel_preprocessor import VoxelPreprocessor
-from opencood.data_utils.pre_processor.bev_preprocessor import BevPreprocessor
-from opencood.data_utils.pre_processor.sp_voxel_preprocessor import SpVoxelPreprocessor
-
-__all__ = {
- "BasePreprocessor": BasePreprocessor,
- "VoxelPreprocessor": VoxelPreprocessor,
- "BevPreprocessor": BevPreprocessor,
- "SpVoxelPreprocessor": SpVoxelPreprocessor,
-}
-
-
-def build_preprocessor(preprocess_cfg, train):
- process_method_name = preprocess_cfg["core_method"]
- error_message = f"{process_method_name} is not found. Please add your processor file's name in opencood/data_utils/processor/init.py"
- assert process_method_name in ["BasePreprocessor", "VoxelPreprocessor", "BevPreprocessor", "SpVoxelPreprocessor"], error_message
-
- processor = __all__[process_method_name](preprocess_params=preprocess_cfg, train=train)
-
- return processor
diff --git a/OpenCOOD/opencood/data_utils/pre_processor/base_preprocessor.py b/OpenCOOD/opencood/data_utils/pre_processor/base_preprocessor.py
deleted file mode 100644
index bb170bf7..00000000
--- a/OpenCOOD/opencood/data_utils/pre_processor/base_preprocessor.py
+++ /dev/null
@@ -1,104 +0,0 @@
-import numpy as np
-import torch
-
-from opencood.utils import pcd_utils
-
-
-class BasePreprocessor(object):
- """
- Basic Lidar pre-processor.
-
- Parameters
- ----------
- preprocess_params : dict
- The dictionary containing all parameters of the preprocessing.
-
- train : bool
- Train or test mode.
- """
-
- def __init__(self, preprocess_params, train):
- self.params = preprocess_params
- self.train = train
-
- def preprocess(self, pcd_np):
- """
- Preprocess the lidar points by simple sampling.
-
- Parameters
- ----------
- pcd_np : np.ndarray
- The raw lidar.
-
- Returns
- -------
- data_dict : the output dictionary.
- """
- data_dict = {}
- sample_num = self.params["args"]["sample_num"]
-
- pcd_np = pcd_utils.downsample_lidar(pcd_np, sample_num)
- data_dict["downsample_lidar"] = pcd_np
-
- return data_dict
-
- @staticmethod
- def collate_batch(batch):
- """Collate sampled point-cloud inputs.
-
- Parameters
- ----------
- batch : list[dict] | dict[str, list]
- Point-cloud inputs represented either as a list of samples or a
- dict of agent input lists.
-
- Returns
- -------
- dict[str, torch.Tensor]
- Batched point clouds under the ``downsample_lidar`` model key.
-
- Raises
- ------
- TypeError
- If ``batch`` is neither a list nor a dictionary.
- """
- if isinstance(batch, list):
- point_clouds = [sample["downsample_lidar"] for sample in batch]
- elif isinstance(batch, dict):
- point_clouds = batch["downsample_lidar"]
- else:
- raise TypeError(f"Unsupported point-cloud batch type: {type(batch).__name__}")
-
- return {"downsample_lidar": torch.from_numpy(np.stack(point_clouds))}
-
- def project_points_to_bev_map(self, points, ratio=0.1):
- """
- Project points to BEV occupancy map with default ratio=0.1.
-
- Parameters
- ----------
- points : np.ndarray
- (N, 3) / (N, 4)
-
- ratio : float
- Discretization parameters. Default is 0.1.
-
- Returns
- -------
- bev_map : np.ndarray
- BEV occupancy map including projected points with shape
- (img_row, img_col).
-
- """
- L1, W1, H1, L2, W2, _ = self.params["cav_lidar_range"]
- img_row = int((L2 - L1) / ratio)
- img_col = int((W2 - W1) / ratio)
- bev_map = np.zeros((img_row, img_col))
- bev_origin = np.array([L1, W1, H1]).reshape(1, -1)
- # (N, 3)
- indices = ((points[:, :3] - bev_origin) / ratio).astype(int)
- mask = np.logical_and(indices[:, 0] > 0, indices[:, 0] < img_row)
- mask = np.logical_and(mask, np.logical_and(indices[:, 1] > 0, indices[:, 1] < img_col))
- indices = indices[mask, :]
- bev_map[indices[:, 0], indices[:, 1]] = 1
- return bev_map
diff --git a/OpenCOOD/opencood/data_utils/pre_processor/bev_preprocessor.py b/OpenCOOD/opencood/data_utils/pre_processor/bev_preprocessor.py
deleted file mode 100644
index b742686e..00000000
--- a/OpenCOOD/opencood/data_utils/pre_processor/bev_preprocessor.py
+++ /dev/null
@@ -1,101 +0,0 @@
-"""
-Convert lidar to bev
-"""
-
-import numpy as np
-import torch
-from opencood.data_utils.pre_processor.base_preprocessor import BasePreprocessor
-
-
-class BevPreprocessor(BasePreprocessor):
- def __init__(self, preprocess_params, train):
- super(BevPreprocessor, self).__init__(preprocess_params, train)
- self.lidar_range = self.params["cav_lidar_range"]
- self.geometry_param = preprocess_params["geometry_param"]
-
- def preprocess(self, pcd_raw):
- """
- Preprocess the lidar points to BEV representations.
-
- Parameters
- ----------
- pcd_raw : np.ndarray
- The raw lidar.
-
- Returns
- -------
- data_dict : the structured output dictionary.
- """
- bev = np.zeros(self.geometry_param["input_shape"], dtype=np.float32)
- intensity_map_count = np.zeros((bev.shape[0], bev.shape[1]), dtype=np.int64)
- bev_origin = np.array([self.geometry_param["L1"], self.geometry_param["W1"], self.geometry_param["H1"]]).reshape(1, -1)
-
- indices = ((pcd_raw[:, :3] - bev_origin) / self.geometry_param["res"]).astype(int)
-
- for i in range(indices.shape[0]):
- bev[indices[i, 0], indices[i, 1], indices[i, 2]] = 1
- bev[indices[i, 0], indices[i, 1], -1] += pcd_raw[i, 3]
- intensity_map_count[indices[i, 0], indices[i, 1]] += 1
- divide_mask = intensity_map_count != 0
- bev[divide_mask, -1] = np.divide(bev[divide_mask, -1], intensity_map_count[divide_mask])
-
- data_dict = {"bev_input": np.transpose(bev, (2, 0, 1))}
- return data_dict
-
- @staticmethod
- def collate_batch_list(batch):
- """
- Customized pytorch data loader collate function.
-
- Parameters
- ----------
- batch : list
- List of dictionary. Each dictionary represent a single frame.
-
- Returns
- -------
- processed_batch : dict
- Updated lidar batch.
- """
- bev_input_list = [x["bev_input"][np.newaxis, ...] for x in batch]
- processed_batch = {"bev_input": torch.from_numpy(np.concatenate(bev_input_list, axis=0))}
- return processed_batch
-
- @staticmethod
- def collate_batch_dict(batch):
- """
- Customized pytorch data loader collate function.
-
- Parameters
- ----------
- batch : dict
- Dict of list. Each element represents a CAV.
-
- Returns
- -------
- processed_batch : dict
- Updated lidar batch.
- """
- bev_input_list = [x[np.newaxis, ...] for x in batch["bev_input"]]
- processed_batch = {"bev_input": torch.from_numpy(np.concatenate(bev_input_list, axis=0))}
- return processed_batch
-
- def collate_batch(self, batch):
- """
- Customized pytorch data loader collate function.
-
- Parameters
- ----------
- batch : list / dict
- Batched data.
- Returns
- -------
- processed_batch : dict
- Updated lidar batch.
- """
- if isinstance(batch, list):
- return self.collate_batch_list(batch)
- elif isinstance(batch, dict):
- return self.collate_batch_dict(batch)
- else:
- raise NotImplementedError
diff --git a/OpenCOOD/opencood/data_utils/pre_processor/sp_voxel_preprocessor.py b/OpenCOOD/opencood/data_utils/pre_processor/sp_voxel_preprocessor.py
deleted file mode 100644
index 10e01c7a..00000000
--- a/OpenCOOD/opencood/data_utils/pre_processor/sp_voxel_preprocessor.py
+++ /dev/null
@@ -1,151 +0,0 @@
-"""
-Transform points to voxels using sparse conv library
-"""
-
-import sys
-
-import numpy as np
-import torch
-from cumm import tensorview as tv
-from opencood.data_utils.pre_processor.base_preprocessor import BasePreprocessor
-
-
-class SpVoxelPreprocessor(BasePreprocessor):
- def __init__(self, preprocess_params, train):
- super(SpVoxelPreprocessor, self).__init__(preprocess_params, train)
- self.spconv = 1
- try:
- # spconv v1.x
- from spconv.utils import VoxelGeneratorV2 as VoxelGenerator
- except ImportError:
- # spconv v2.x
- from spconv.utils import Point2VoxelCPU3d as VoxelGenerator
-
- self.spconv = 2
- self.lidar_range = self.params["cav_lidar_range"]
- self.voxel_size = self.params["args"]["voxel_size"]
- self.max_points_per_voxel = self.params["args"]["max_points_per_voxel"]
-
- if train:
- self.max_voxels = self.params["args"]["max_voxel_train"]
- else:
- self.max_voxels = self.params["args"]["max_voxel_test"]
-
- grid_size = (np.array(self.lidar_range[3:6]) - np.array(self.lidar_range[0:3])) / np.array(self.voxel_size)
- self.grid_size = np.round(grid_size).astype(np.int64)
-
- # use sparse conv library to generate voxel
- if self.spconv == 1:
- self.voxel_generator = VoxelGenerator(
- voxel_size=self.voxel_size, point_cloud_range=self.lidar_range, max_num_points=self.max_points_per_voxel, max_voxels=self.max_voxels
- )
- else:
- self.voxel_generator = VoxelGenerator(
- vsize_xyz=self.voxel_size,
- coors_range_xyz=self.lidar_range,
- max_num_points_per_voxel=self.max_points_per_voxel,
- num_point_features=4,
- max_num_voxels=self.max_voxels,
- )
-
- def preprocess(self, pcd_np):
- data_dict = {}
- if self.spconv == 1:
- voxel_output = self.voxel_generator.generate(pcd_np)
- else:
- pcd_tv = tv.from_numpy(pcd_np)
- voxel_output = self.voxel_generator.point_to_voxel(pcd_tv)
- if isinstance(voxel_output, dict):
- voxels, coordinates, num_points = voxel_output["voxels"], voxel_output["coordinates"], voxel_output["num_points_per_voxel"]
- else:
- voxels, coordinates, num_points = voxel_output
-
- if self.spconv == 2:
- voxels = voxels.numpy()
- coordinates = coordinates.numpy()
- num_points = num_points.numpy()
-
- data_dict["voxel_features"] = voxels
- data_dict["voxel_coords"] = coordinates
- data_dict["voxel_num_points"] = num_points
-
- return data_dict
-
- def collate_batch(self, batch):
- """
- Customized pytorch data loader collate function.
-
- Parameters
- ----------
- batch : list or dict
- List or dictionary.
-
- Returns
- -------
- processed_batch : dict
- Updated lidar batch.
- """
-
- if isinstance(batch, list):
- return self.collate_batch_list(batch)
- elif isinstance(batch, dict):
- return self.collate_batch_dict(batch)
- else:
- sys.exit("Batch has too be a list or a dictionarn")
-
- @staticmethod
- def collate_batch_list(batch):
- """
- Customized pytorch data loader collate function.
-
- Parameters
- ----------
- batch : list
- List of dictionary. Each dictionary represent a single frame.
-
- Returns
- -------
- processed_batch : dict
- Updated lidar batch.
- """
- voxel_features = []
- voxel_num_points = []
- voxel_coords = []
-
- for i in range(len(batch)):
- voxel_features.append(batch[i]["voxel_features"])
- voxel_num_points.append(batch[i]["voxel_num_points"])
- coords = batch[i]["voxel_coords"]
- voxel_coords.append(np.pad(coords, ((0, 0), (1, 0)), mode="constant", constant_values=i))
-
- voxel_num_points = torch.from_numpy(np.concatenate(voxel_num_points))
- voxel_features = torch.from_numpy(np.concatenate(voxel_features))
- voxel_coords = torch.from_numpy(np.concatenate(voxel_coords))
-
- return {"voxel_features": voxel_features, "voxel_coords": voxel_coords, "voxel_num_points": voxel_num_points}
-
- @staticmethod
- def collate_batch_dict(batch: dict):
- """
- Collate batch if the batch is a dictionary,
- eg: {'voxel_features': [feature1, feature2...., feature n]}
-
- Parameters
- ----------
- batch : dict
-
- Returns
- -------
- processed_batch : dict
- Updated lidar batch.
- """
- voxel_features = torch.from_numpy(np.concatenate(batch["voxel_features"]))
- voxel_num_points = torch.from_numpy(np.concatenate(batch["voxel_num_points"]))
- coords = batch["voxel_coords"]
- voxel_coords = []
-
- for i in range(len(coords)):
- voxel_coords.append(np.pad(coords[i], ((0, 0), (1, 0)), mode="constant", constant_values=i))
- voxel_coords = torch.from_numpy(np.concatenate(voxel_coords))
-
- return {"voxel_features": voxel_features, "voxel_coords": voxel_coords, "voxel_num_points": voxel_num_points}
diff --git a/OpenCOOD/opencood/data_utils/pre_processor/voxel_preprocessor.py b/OpenCOOD/opencood/data_utils/pre_processor/voxel_preprocessor.py
deleted file mode 100644
index 4d487844..00000000
--- a/OpenCOOD/opencood/data_utils/pre_processor/voxel_preprocessor.py
+++ /dev/null
@@ -1,140 +0,0 @@
-"""
-Convert lidar to voxel. This class was manually designed, and we recommend
-to use sp_voxel_preprocessor.
-"""
-
-import sys
-
-import numpy as np
-import torch
-
-from opencood.data_utils.pre_processor.base_preprocessor import BasePreprocessor
-
-
-class VoxelPreprocessor(BasePreprocessor):
- def __init__(self, preprocess_params, train):
- super(VoxelPreprocessor, self).__init__(preprocess_params, train)
- # TODO: add intermediate lidar range later
- self.lidar_range = self.params["cav_lidar_range"]
-
- self.vw = self.params["args"]["vw"]
- self.vh = self.params["args"]["vh"]
- self.vd = self.params["args"]["vd"]
- self.T = self.params["args"]["T"]
-
- def preprocess(self, pcd_np):
- """
- Preprocess the lidar points by voxelization.
-
- Parameters
- ----------
- pcd_np : np.ndarray
- The raw lidar.
-
- Returns
- -------
- data_dict : the structured output dictionary.
- """
- data_dict = {}
-
- # calculate the voxel coordinates
- voxel_coords = (
- pcd_np[:, :3] - np.floor(np.array([self.lidar_range[0], self.lidar_range[1], self.lidar_range[2]])) / (self.vw, self.vh, self.vd)
- ).astype(np.int32)
-
- # convert to (D, H, W) as the paper
- voxel_coords = voxel_coords[:, [2, 1, 0]]
- voxel_coords, inv_ind, voxel_counts = np.unique(voxel_coords, axis=0, return_inverse=True, return_counts=True)
-
- voxel_features = []
-
- for i in range(len(voxel_coords)):
- voxel = np.zeros((self.T, 7), dtype=np.float32)
- pts = pcd_np[inv_ind == i]
- if voxel_counts[i] > self.T:
- pts = pts[: self.T, :]
- voxel_counts[i] = self.T
-
- # augment the points
- voxel[: pts.shape[0], :] = np.concatenate((pts, pts[:, :3] - np.mean(pts[:, :3], 0)), axis=1)
- voxel_features.append(voxel)
-
- data_dict["voxel_features"] = np.array(voxel_features)
- data_dict["voxel_coords"] = voxel_coords
-
- return data_dict
-
- def collate_batch(self, batch):
- """
- Customized pytorch data loader collate function.
-
- Parameters
- ----------
- batch : list or dict
- List or dictionary.
-
- Returns
- -------
- processed_batch : dict
- Updated lidar batch.
- """
-
- if isinstance(batch, list):
- return self.collate_batch_list(batch)
- elif isinstance(batch, dict):
- return self.collate_batch_dict(batch)
- else:
- sys.exit("Batch has too be a list or a dictionarn")
-
- @staticmethod
- def collate_batch_list(batch):
- """
- Customized pytorch data loader collate function.
-
- Parameters
- ----------
- batch : list
- List of dictionary. Each dictionary represent a single frame.
-
- Returns
- -------
- processed_batch : dict
- Updated lidar batch.
- """
- voxel_features = []
- voxel_coords = []
-
- for i in range(len(batch)):
- voxel_features.append(batch[i]["voxel_features"])
- coords = batch[i]["voxel_coords"]
- voxel_coords.append(np.pad(coords, ((0, 0), (1, 0)), mode="constant", constant_values=i))
-
- voxel_features = torch.from_numpy(np.concatenate(voxel_features))
- voxel_coords = torch.from_numpy(np.concatenate(voxel_coords))
-
- return {"voxel_features": voxel_features, "voxel_coords": voxel_coords}
-
- @staticmethod
- def collate_batch_dict(batch: dict):
- """
- Collate batch if the batch is a dictionary,
- eg: {'voxel_features': [feature1, feature2...., feature n]}
-
- Parameters
- ----------
- batch : dict
-
- Returns
- -------
- processed_batch : dict
- Updated lidar batch.
- """
- voxel_features = torch.from_numpy(np.concatenate(batch["voxel_features"]))
- coords = batch["voxel_coords"]
- voxel_coords = []
-
- for i in range(len(coords)):
- voxel_coords.append(np.pad(coords[i], ((0, 0), (1, 0)), mode="constant", constant_values=i))
- voxel_coords = torch.from_numpy(np.concatenate(voxel_coords))
-
- return {"voxel_features": voxel_features, "voxel_coords": voxel_coords}
diff --git a/OpenCOOD/opencood/hypes_yaml/__init__.py b/OpenCOOD/opencood/hypes_yaml/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/OpenCOOD/opencood/hypes_yaml/ciassd_early_fusion.yaml b/OpenCOOD/opencood/hypes_yaml/ciassd_early_fusion.yaml
deleted file mode 100644
index d69b5eca..00000000
--- a/OpenCOOD/opencood/hypes_yaml/ciassd_early_fusion.yaml
+++ /dev/null
@@ -1,120 +0,0 @@
-name: ciassd_early_fusion
-
-yaml_parser: "load_voxel_params"
-root_dir: "/media/hdd/yuan/koko/data/OPV2V/train" # "/media/hdd/yuan/data/OPV2V/train"
-validate_dir: "/media/hdd/yuan/koko/data/OPV2V/validate"
-
-train_params:
- batch_size: &batch_size 6
- epoches: 100
- eval_freq: 2
- save_freq: 2
-
-fusion:
- core_method: 'EarlyFusionDataset' # LateFusionDataset, EarlyFusionDataset, IntermediateFusionDataset supported
- args: []
-
-# preprocess-related
-preprocess:
- # options: BasePreprocessor, VoxelPreprocessor, BevPreprocessor
- core_method: 'SpVoxelPreprocessor'
- args:
- voxel_size: &voxel_size [0.1, 0.1, 0.1]
- max_points_per_voxel: &T 5
- max_voxel_train: 36000
- max_voxel_test: 70000
- # lidar range for each individual cav.
- cav_lidar_range: &cav_lidar [-140.8, -40, -3, 140.8, 40, 1]
-
-data_augment:
- - NAME: random_world_flip
- ALONG_AXIS_LIST: [ 'x' ]
-
- - NAME: random_world_rotation
- WORLD_ROT_ANGLE: [ -0.78539816, 0.78539816 ]
-
- - NAME: random_world_scaling
- WORLD_SCALE_RANGE: [ 0.95, 1.05 ]
-
-# anchor box related
-postprocess:
- core_method: 'CiassdPostprocessor' # VoxelPostprocessor, BevPostprocessor supported
- predict_iou: true
- anchor_args:
- cav_lidar_range: *cav_lidar
- l: 3.9
- w: 1.6
- h: 1.56
- r: [0, 90]
- num: &achor_num 2
- feature_stride: 8
- target_args:
- pos_threshold: 0.6
- neg_threshold: 0.45
- score_threshold: 0.3
- order: 'hwl' # hwl or lwh
- max_num: 100 # maximum number of objects in a single frame. use this number to make sure different frames has the same dimension in the same batch
- nms_thresh: 0.2
-
-# model related
-model:
- core_method: ciassd # corresponding to VoxelNet
- args:
- N: *batch_size
- T: *T
- lidar_range: *cav_lidar
- voxel_size: *voxel_size
- anchor_num: *achor_num
- mean_vfe:
- num_point_features: 4
- spconv:
- num_features_in: 64
- num_features_out: 64
- map2bev:
- feature_num: 128
- ssfa:
- feature_num: 128
- head:
- num_input: 128
- num_pred: 14
- num_cls: 2
- num_iou: 2
- use_dir: True
- num_dir: 4
-
-loss:
- core_method: ciassd_loss
- args:
- pos_cls_weight: 50.0
- encode_rad_error_by_sin: true
- cls:
- type: 'SigmoidFocalLoss'
- alpha: 0.25
- gamma: 2.0
- weight: 2.0
- reg:
- type: 'WeightedSmoothL1Loss'
- sigma: 3.0
- codewise: true
- weight: 2.0
- iou:
- type: 'WeightedSmoothL1Loss'
- sigma: 3.0
- codewise: true
- weight: 1.0
- dir:
- type: 'WeightedSoftmaxClassificationLoss'
- weight: 0.2
-
-
-optimizer:
- core_method: Adam
- lr: 0.002
- args:
- eps: 1e-10
- weight_decay: 1e-4
-
-lr_scheduler:
- core_method: multistep #step, multistep and exponential and cosineannealwarm support
- gamma: 0.1
- step_size: [15, 30]
diff --git a/OpenCOOD/opencood/hypes_yaml/ciassd_intermediate_fusion.yaml b/OpenCOOD/opencood/hypes_yaml/ciassd_intermediate_fusion.yaml
deleted file mode 100644
index d03f6c47..00000000
--- a/OpenCOOD/opencood/hypes_yaml/ciassd_intermediate_fusion.yaml
+++ /dev/null
@@ -1,89 +0,0 @@
-name: ciassd
-
-yaml_parser: "load_voxel_params"
-root_dir: "/media/hdd/yuan/data/OPV2V/train"
-validate_dir: "/media/hdd/yuan/data/OPV2V/validate"
-
-train_params:
- batch_size: &batch_size 1
- epoches: 30
- eval_freq: 1
- save_freq: 1
-
-fusion:
- core_method: 'IntermediateFusionDataset' # LateFusionDataset, EarlyFusionDataset, IntermediateFusionDataset supported
- args: []
-
-# preprocess-related
-preprocess:
- # options: BasePreprocessor, VoxelPreprocessor, BevPreprocessor
- core_method: 'SpVoxelPreprocessor'
- args:
- voxel_size: &voxel_size [0.4, 0.4, 0.4]
- max_points_per_voxel: &T 32
- max_voxel_train: 36000
- max_voxel_test: 70000
- # lidar range for each individual cav.
- cav_lidar_range: &cav_lidar [-140.8, -41.6, -3, 140.8, 41.6, 1]
-
-data_augment:
- - NAME: random_world_flip
- ALONG_AXIS_LIST: [ 'x' ]
-
- - NAME: random_world_rotation
- WORLD_ROT_ANGLE: [ -0.78539816, 0.78539816 ]
-
- - NAME: random_world_scaling
- WORLD_SCALE_RANGE: [ 0.95, 1.05 ]
-
-# anchor box related
-postprocess:
- core_method: 'VoxelPostprocessor' # VoxelPostprocessor, BevPostprocessor supported
- anchor_args:
- cav_lidar_range: *cav_lidar
- l: 3.9
- w: 1.6
- h: 1.56
- r: [0, 90]
- num: &achor_num 2
- target_args:
- pos_threshold: 0.6
- neg_threshold: 0.45
- score_threshold: 0.25
- order: 'hwl' # hwl or lwh
- max_num: 100 # maximum number of objects in a single frame. use this number to make sure different frames has the same dimension in the same batch
- nms_thresh: 0.15
-
-# model related
-model:
- core_method: voxel_net_intermediate # corresponding to VoxelNet
- args:
- N: *batch_size
- T: *T
- lidar_range: *cav_lidar
- voxel_size: *voxel_size
- anchor_num: *achor_num
- compression: 4
- pillar_vfe:
- use_norm: true
- with_distance: false
- use_absolute_xyz: true
- num_filters: [ 64 ]
-
-loss:
- core_method: point_pillar_loss
- args:
- cls_weight: 1.0
- reg: 2.0
-
-optimizer:
- core_method: Adam
- lr: 0.002
- args:
- eps: 1e-10
- weight_decay: 1e-4
-
-lr_scheduler:
- core_method: multistep #step, multistep and exponential and cosineannealwarm support
- gamma: 0.1
- step_size: [15, 30]
diff --git a/OpenCOOD/opencood/hypes_yaml/ciassd_late_fusion.yaml b/OpenCOOD/opencood/hypes_yaml/ciassd_late_fusion.yaml
deleted file mode 100644
index d3d2ef2a..00000000
--- a/OpenCOOD/opencood/hypes_yaml/ciassd_late_fusion.yaml
+++ /dev/null
@@ -1,120 +0,0 @@
-name: ciassd_late_fusion
-
-yaml_parser: "load_voxel_params"
-root_dir: "opv2v_data_dumping/train"
-validate_dir: "opv2v_data_dumping/validate"
-
-train_params:
- batch_size: &batch_size 6
- epoches: 100
- eval_freq: 2
- save_freq: 2
-
-fusion:
- core_method: 'LateFusionDataset' # LateFusionDataset, EarlyFusionDataset, IntermediateFusionDataset supported
- args: []
-
-# preprocess-related
-preprocess:
- # options: BasePreprocessor, VoxelPreprocessor, BevPreprocessor
- core_method: 'SpVoxelPreprocessor'
- args:
- voxel_size: &voxel_size [0.1, 0.1, 0.1]
- max_points_per_voxel: &T 5
- max_voxel_train: 36000
- max_voxel_test: 70000
- # lidar range for each individual cav.
- cav_lidar_range: &cav_lidar [-140.8, -40, -8, 140.8, 40, 8]
-
-data_augment:
- - NAME: random_world_flip
- ALONG_AXIS_LIST: [ 'x' ]
-
- - NAME: random_world_rotation
- WORLD_ROT_ANGLE: [ -0.78539816, 0.78539816 ]
-
- - NAME: random_world_scaling
- WORLD_SCALE_RANGE: [ 0.95, 1.05 ]
-
-# anchor box related
-postprocess:
- core_method: 'CiassdPostprocessor' # VoxelPostprocessor, BevPostprocessor supported
- predict_iou: true
- anchor_args:
- cav_lidar_range: *cav_lidar
- l: 3.9
- w: 1.6
- h: 1.56
- r: [0, 90]
- num: &achor_num 2
- feature_stride: 8
- target_args:
- pos_threshold: 0.6
- neg_threshold: 0.45
- score_threshold: 0.3
- order: 'hwl' # hwl or lwh
- max_num: 100 # maximum number of objects in a single frame. use this number to make sure different frames has the same dimension in the same batch
- nms_thresh: 0.2
-
-# model related
-model:
- core_method: ciassd # corresponding to VoxelNet
- args:
- N: *batch_size
- T: *T
- lidar_range: *cav_lidar
- voxel_size: *voxel_size
- anchor_num: *achor_num
- mean_vfe:
- num_point_features: 4
- spconv:
- num_features_in: 64
- num_features_out: 64
- map2bev:
- feature_num: 128
- ssfa:
- feature_num: 128
- head:
- num_input: 128
- num_pred: 14
- num_cls: 2
- num_iou: 2
- use_dir: True
- num_dir: 4
-
-loss:
- core_method: ciassd_loss
- args:
- pos_cls_weight: 50.0
- encode_rad_error_by_sin: true
- cls:
- type: 'SigmoidFocalLoss'
- alpha: 0.25
- gamma: 2.0
- weight: 2.0
- reg:
- type: 'WeightedSmoothL1Loss'
- sigma: 3.0
- codewise: true
- weight: 2.0
- iou:
- type: 'WeightedSmoothL1Loss'
- sigma: 3.0
- codewise: true
- weight: 1.0
- dir:
- type: 'WeightedSoftmaxClassificationLoss'
- weight: 0.2
-
-
-optimizer:
- core_method: Adam
- lr: 0.002
- args:
- eps: 1e-10
- weight_decay: 1e-4
-
-lr_scheduler:
- core_method: multistep #step, multistep and exponential and cosineannealwarm support
- gamma: 0.1
- step_size: [15, 30]
diff --git a/OpenCOOD/opencood/hypes_yaml/fpvrcnn_intermediate_fusion.yaml b/OpenCOOD/opencood/hypes_yaml/fpvrcnn_intermediate_fusion.yaml
deleted file mode 100644
index 261df692..00000000
--- a/OpenCOOD/opencood/hypes_yaml/fpvrcnn_intermediate_fusion.yaml
+++ /dev/null
@@ -1,177 +0,0 @@
-name: fpvrcnn_intermediate_fusion
-
-yaml_parser: "load_voxel_params"
-root_dir: "opv2v_data_dumping/train"
-validate_dir: "opv2v_data_dumping/validate"
-
-train_params:
- batch_size: &batch_size 4
- epoches: 20
- eval_freq: 1
- save_freq: 1
- max_cav: &max_cav 10
-
-fusion:
- core_method: 'IntermediateFusionDatasetV2' # LateFusionDataset, EarlyFusionDataset, IntermediateFusionDataset IntermediateFusionDatasetV2 supported
- args: []
-
-# preprocess-related
-preprocess:
- # options: BasePreprocessor, VoxelPreprocessor, BevPreprocessor
- core_method: 'SpVoxelPreprocessor'
- args:
- voxel_size: &voxel_size [0.1, 0.1, 0.4]
- max_points_per_voxel: &T 5
- max_voxel_train: 36000
- max_voxel_test: 70000
- # lidar range for each individual cav.
- cav_lidar_range: &cav_lidar [-140.8, -41.6, -10.6, 140.8, 41.6, 10.6]
- keep_original_lidar: true
-
-data_augment:
- - NAME: random_world_flip
- ALONG_AXIS_LIST: [ 'x' ]
-
- - NAME: random_world_rotation
- WORLD_ROT_ANGLE: [ -0.78539816, 0.78539816 ]
-
- - NAME: random_world_scaling
- WORLD_SCALE_RANGE: [ 0.95, 1.05 ]
-
-# anchor box related
-postprocess: &postprocess_params
- core_method: 'FpvrcnnPostprocessor' # VoxelPostprocessor, BevPostprocessor, FpvrcnnPostprocessor supported
- anchor_args:
- cav_lidar_range: *cav_lidar
- l: 3.9
- w: 1.6
- h: 1.56
- r: [0, 90]
- feature_stride: 8
- num: &achor_num 2
- target_args:
- pos_threshold: 0.6
- neg_threshold: 0.45
- score_threshold: 0.5
- order: 'hwl' # hwl or lwh
- max_num: 100 # maximum number of objects in a single frame. use this number to make sure different frames has the same dimension in the same batch
- nms_thresh: 0.15
-
-# model related
-model:
- core_method: fpvrcnn # corresponding to FPVRCNN
- args:
- activate_stage2: False # True after training for first 20 epochs
- N: *batch_size
- T: *T
- lidar_range: *cav_lidar
- voxel_size: *voxel_size
- anchor_num: *achor_num
- mean_vfe:
- num_point_features: 4
- spconv:
- num_features_in: 4
- num_features_out: 64
- map2bev:
- feature_num: 128
- ssfa:
- feature_num: 128
- head:
- num_input: 128
- num_pred: 14
- num_cls: 2
- num_iou: 2
- use_dir: True
- num_dir: 4
- post_processer: *postprocess_params
- vsa:
- enlarge_selection_boxes: true
- point_source: 'raw_points'
- num_keypoints: 4096
- num_out_features: &cpm_features 32
- features_source: ['bev', 'x_conv1', 'x_conv2', 'x_conv3', 'x_conv4', 'raw_points']
- sa_layer:
- raw_points:
- mlps: [[16, 16], [16, 16]]
- pool_radius: [0.4, 0.8]
- n_sample: [16, 16]
- x_conv1:
- downsample_factor: 1
- mlps: [[16, 16], [16, 16]]
- pool_radius: [0.4, 0.8]
- n_sample: [16, 16]
- x_conv2:
- downsample_factor: 2
- mlps: [[32, 32], [32, 32]]
- pool_radius: [0.8, 1.2]
- n_sample: [16, 32]
- x_conv3:
- downsample_factor: 4
- mlps: [[64, 64], [64, 64]]
- pool_radius: [1.2, 2.4]
- n_sample: [16, 32]
- x_conv4:
- downsample_factor: 8
- mlps: [[64, 64], [64, 64]]
- pool_radius: [2.4, 4.8]
- n_sample: [16, 32]
- matcher: 'none' # may need to configure parameters for loc correction
- roi_head:
- num_cls: 1
- in_channels: *cpm_features
- n_fc_neurons: 256
- dp_ratio: 0.3 # dropout ratio
- roi_grid_pool:
- grid_size: 6
- mlps: [[64, 64], [64, 64]]
- pool_radius: [0.8, 1.6]
- n_sample: [16, 16]
- pool_method: 'max_pool'
-
-loss:
- core_method: fpvrcnn_loss
- args:
- stage1:
- pos_cls_weight: 2.0
- encode_rad_error_by_sin: true
- cls:
- type: 'SigmoidFocalLoss'
- alpha: 0.25
- gamma: 2.0
- weight: 2.0
- reg:
- type: 'WeightedSmoothL1Loss'
- sigma: 3.0
- codewise: true
- weight: 2.0
- iou: &iou
- type: 'WeightedSmoothL1Loss'
- sigma: 3.0
- codewise: true
- weight: 1.0
- dir:
- type: 'WeightedSoftmaxClassificationLoss'
- weight: 0.2
- stage2:
- cls:
- type: 'WeightedSigmoidBinaryCELoss'
- weight: 1.0
- iou: *iou
- reg:
- type: 'WeightedSmoothL1Loss'
- sigma: 3.0
- reduction: 'mean'
- code_weights: [20.0]
- weight: 1.0
-
-optimizer:
- core_method: Adam
- lr: 0.002
- args:
- eps: 1e-10
- weight_decay: 1e-4
-
-lr_scheduler:
- core_method: multistep #step, multistep and exponential and cosineannealwarm support
- gamma: 0.1
- step_size: [10, 20]
diff --git a/OpenCOOD/opencood/hypes_yaml/pixor_early_fusion.yaml b/OpenCOOD/opencood/hypes_yaml/pixor_early_fusion.yaml
deleted file mode 100644
index f3ac4586..00000000
--- a/OpenCOOD/opencood/hypes_yaml/pixor_early_fusion.yaml
+++ /dev/null
@@ -1,64 +0,0 @@
-name: pixor_early_fusion
-
-yaml_parser: "load_bev_params"
-root_dir: "opv2v_data_dumping/train"
-validate_dir: "opv2v_data_dumping/validate"
-train_params:
- batch_size: &batch_size 2
- epoches: 15
- eval_freq: 1
- save_freq: 1
-fusion:
- core_method: 'EarlyFusionDataset' # LateFusionDataset, EarlyFusionDataset, IntermediateFusionDataset supported
- args: []
-
-# preprocess-related
-preprocess:
- # options: BasePreprocessor, VoxelPreprocessor, BevPreprocessor
- core_method: 'BevPreprocessor'
- args:
- res: &res 0.2 # discretization resolusion
- downsample_rate: &downsample_rate 4 # pixor downsample ratio
- # lidar range for each individual cav.
- cav_lidar_range: &cav_lidar [-160, -40, -3, 160, 40, 1] # must be divisible by 16
-
-data_augment: []
-
-# anchor box related
-postprocess:
- core_method: 'BevPostprocessor' # VoxelPostprocessor, BevPostprocessor supported
- nms_thresh: 0.15
- anchor_args:
- cav_lidar_range: *cav_lidar
- res: *res
- downsample_rate: *downsample_rate # pixor downsample ratio
- target_args:
- score_threshold: 0.5
-
-
- order: 'lwh' # hwl or lwh
- max_num: 100 # maximum number of objects in a single frame. use this number to make sure different frames has the same dimension in the same batch
-
-# model related
-model:
- core_method: pixor # corresponding to VoxelNet
- args:
- use_bn: True
- decode: False
-
-loss:
- core_method: pixor_loss
- args:
- alpha: 1.0
- beta: 1.0
-
-optimizer:
- core_method: Adam
- lr: 0.001
- args:
- eps: 1e-10
- weight_decay: 1e-4
-
-lr_scheduler:
- core_method: exponential #step, multistep and exponential and cosineannealwarm support
- gamma: 0.99
diff --git a/OpenCOOD/opencood/hypes_yaml/pixor_intermediate_fusion.yaml b/OpenCOOD/opencood/hypes_yaml/pixor_intermediate_fusion.yaml
deleted file mode 100644
index 3061efb8..00000000
--- a/OpenCOOD/opencood/hypes_yaml/pixor_intermediate_fusion.yaml
+++ /dev/null
@@ -1,69 +0,0 @@
-name: pixor_intermediate_fusion
-
-yaml_parser: "load_bev_params"
-root_dir: "opv2v_data_dumping/train"
-validate_dir: "opv2v_data_dumping/validate"
-train_params:
- batch_size: &batch_size 2
- epoches: 10
- eval_freq: 1
- save_freq: 1
-fusion:
- core_method: 'IntermediateFusionDataset' # LateFusionDataset, EarlyFusionDataset, IntermediateFusionDataset supported
- args: []
-
-# preprocess-related
-preprocess:
- # options: BasePreprocessor, VoxelPreprocessor, BevPreprocessor
- core_method: 'BevPreprocessor'
- args:
- res: &res 0.5 # discretization resolusion
- downsample_rate: &downsample_rate 4 # pixor downsample ratio
- # lidar range for each individual cav.
- cav_lidar_range: &cav_lidar [-160, -40, -8, 160, 40, 8] # must be divisible by 16
-
-data_augment: []
-
-# anchor box related
-postprocess:
- core_method: 'BevPostprocessor' # VoxelPostprocessor, BevPostprocessor supported
- nms_thresh: 0.15
- anchor_args:
- cav_lidar_range: *cav_lidar
- res: *res
- downsample_rate: *downsample_rate # pixor downsample ratio
- target_args:
- score_threshold: 0.7
-
-
- order: 'lwh' # hwl or lwh
- max_num: 100 # maximum number of objects in a single frame. use this number to make sure different frames has the same dimension in the same batch
-
-# model related
-model:
- core_method: pixor_intermediate
- args:
- use_bn: True
- decode: False
-
-loss:
- core_method: pixor_loss
- args:
- alpha: 1.0
- beta: 1.0
-
-optimizer:
- core_method: Adam
- lr: 0.00001
- args:
- eps: 1e-10
- weight_decay: 1e-4
-
-lr_scheduler:
- core_method: cosineannealwarm #step, multistep and exponential and cosineannealwarm support
- epoches: 1 # Specifying the number of epochs that matches train_params
- T_0: 5 # The initial period for CosineAnnealingWarmRestarts
- T_mult: 1 # Period Multiplier
- warmup_lr: 0.00001 # Initial learning rate for warming up
- warmup_epoches: 5 # Number of epochs to warm up
- lr_min: 0.00001 # The minimum value of the learning rate
diff --git a/OpenCOOD/opencood/hypes_yaml/pixor_late_fusion.yaml b/OpenCOOD/opencood/hypes_yaml/pixor_late_fusion.yaml
deleted file mode 100644
index 59064426..00000000
--- a/OpenCOOD/opencood/hypes_yaml/pixor_late_fusion.yaml
+++ /dev/null
@@ -1,58 +0,0 @@
-name: pixor_post_fusion
-
-yaml_parser: "load_bev_params"
-root_dir: "opv2v_data_dumping/train"
-validate_dir: "opv2v_data_dumping/validate"
-train_params:
- batch_size: &batch_size 2
- epoches: 15
- eval_freq: 1
- save_freq: 1
-fusion:
- core_method: 'LateFusionDataset' # LateFusionDataset, EarlyFusionDataset, IntermediateFusionDataset supported
- args: []
-
-# preprocess-related
-preprocess:
- # options: BasePreprocessor, VoxelPreprocessor, BevPreprocessor
- core_method: 'BevPreprocessor'
- args:
- res: &res 0.2 # discretization resolusion
- downsample_rate: &downsample_rate 4 # pixor downsample ratio
- # lidar range for each individual cav.
- cav_lidar_range: &cav_lidar [-80, -40, -3, 80, 40, 1] # must be divisible by 16
-
-# anchor box related
-postprocess:
- core_method: 'BevPostprocessor' # VoxelPostprocessor, BevPostprocessor supported
- nms_thresh: 0.15
- anchor_args:
- cav_lidar_range: *cav_lidar
- res: *res
- downsample_rate: *downsample_rate # pixor downsample ratio
- target_args:
- score_threshold: 0.3
-
-
- order: 'lwh' # hwl or lwh
- max_num: 100 # maximum number of objects in a single frame. use this number to make sure different frames has the same dimension in the same batch
-
-# model related
-model:
- core_method: pixor # corresponding to VoxelNet
- args:
- use_bn: True
- decode: False
-
-loss:
- core_method: pixor_loss
- args:
- alpha: 1.0
- beta: 1.0
-
-optimizer:
- core_method: Adam
- lr: 0.001
- args:
- eps: 1e-10
- weight_decay: 1e-4
diff --git a/OpenCOOD/opencood/hypes_yaml/point_pillar_coalign.yaml b/OpenCOOD/opencood/hypes_yaml/point_pillar_coalign.yaml
deleted file mode 100644
index cdc496ba..00000000
--- a/OpenCOOD/opencood/hypes_yaml/point_pillar_coalign.yaml
+++ /dev/null
@@ -1,105 +0,0 @@
-# This yaml only contains CoAlign's multiscale intermediate feature fusion.
-# If you want build the agent-object pose graph to correct pose error,
-# please refer to https://github.com/yifanlu0227/CoAlign
-
-name: point_pillar_coalign
-root_dir: "opv2v_data_dumping/train"
-validate_dir: "opv2v_data_dumping/validate"
-
-
-yaml_parser: "load_point_pillar_params"
-train_params:
- batch_size: &batch_size 2
- epoches: 15
- eval_freq: 1
- save_freq: 1
-
-fusion:
- core_method: 'IntermediateFusionDataset' # LateFusionDataset, EarlyFusionDataset, IntermediateFusionDataset supported
- args: []
-
-# preprocess-related
-preprocess:
- # options: BasePreprocessor, VoxelPreprocessor, BevPreprocessor
- core_method: 'SpVoxelPreprocessor'
- args:
- voxel_size: &voxel_size [0.4, 0.4, 4]
- max_points_per_voxel: 32
- max_voxel_train: 32000
- max_voxel_test: 70000
- # lidar range for each individual cav.
- cav_lidar_range: &cav_lidar [-140.8, -40, -3, 140.8, 40, 1]
-
-data_augment:
- - NAME: random_world_flip
- ALONG_AXIS_LIST: [ 'x' ]
-
- - NAME: random_world_rotation
- WORLD_ROT_ANGLE: [ -0.78539816, 0.78539816 ]
-
- - NAME: random_world_scaling
- WORLD_SCALE_RANGE: [ 0.95, 1.05 ]
-
-# anchor box related
-postprocess:
- core_method: 'VoxelPostprocessor' # VoxelPostprocessor, BevPostprocessor supported
- anchor_args:
- cav_lidar_range: *cav_lidar
- l: 3.9
- w: 1.6
- h: 1.56
- r: [0, 90]
- feature_stride: 2
- num: &achor_num 2
- target_args:
- pos_threshold: 0.6
- neg_threshold: 0.45
- score_threshold: 0.20
- order: 'hwl' # hwl or lwh
- max_num: 100 # maximum number of objects in a single frame. use this number to make sure different frames has the same dimension in the same batch
- nms_thresh: 0.15
-
-# model related
-model:
- core_method: point_pillar_coalign
- args:
- voxel_size: *voxel_size
- lidar_range: *cav_lidar
- anchor_number: *achor_num
- compression: 0 # compression rate
-
- pillar_vfe:
- use_norm: true
- with_distance: false
- use_absolute_xyz: true
- num_filters: [64]
- point_pillar_scatter:
- num_features: 64
-
- res_bev_backbone:
- layer_nums: [3, 5, 8]
- layer_strides: [2, 2, 2]
- num_filters: [64, 128, 256]
- upsample_strides: [1, 2, 4]
- num_upsample_filter: [128, 128, 128]
- compression: 0
-
- anchor_num: *achor_num
-
-loss:
- core_method: point_pillar_loss
- args:
- cls_weight: 1.0
- reg: 2.0
-
-optimizer:
- core_method: Adam
- lr: 0.002
- args:
- eps: 1e-10
- weight_decay: 1e-4
-
-lr_scheduler:
- core_method: multistep #step, multistep and exponential and cosineannealwarm support
- gamma: 0.1
- step_size: [10, 15]
diff --git a/OpenCOOD/opencood/hypes_yaml/point_pillar_cobevt.yaml b/OpenCOOD/opencood/hypes_yaml/point_pillar_cobevt.yaml
deleted file mode 100644
index 5e7fdc9c..00000000
--- a/OpenCOOD/opencood/hypes_yaml/point_pillar_cobevt.yaml
+++ /dev/null
@@ -1,132 +0,0 @@
-name: corpbevtlidar
-root_dir: 'v2xset/train'
-validate_dir: 'v2xset/validate'
-
-wild_setting:
- async: false
- async_overhead: 100
- seed: 20
- loc_err: false
- xyz_std: 0.2
- ryp_std: 0.2
- data_size: 1.06 # Mb!!
- transmission_speed: 27 # Mbps!!
- backbone_delay: 10 # ms
-
-yaml_parser: "load_point_pillar_params"
-train_params:
- batch_size: &batch_size 4
- epoches: &epoches 90
- eval_freq: 2
- save_freq: 1
- max_cav: &max_cav 5
-
-fusion:
- core_method: 'IntermediateFusionDataset' # LateFusionDataset, EarlyFusionDataset, IntermediateFusionDataset supported
- args: []
-
-# preprocess-related
-preprocess:
- # options: BasePreprocessor, VoxelPreprocessor, BevPreprocessor
- core_method: 'SpVoxelPreprocessor'
- args:
- voxel_size: &voxel_size [0.4, 0.4, 4]
- max_points_per_voxel: 32
- max_voxel_train: 32000
- max_voxel_test: 70000
- # lidar range for each individual cav.
- cav_lidar_range: &cav_lidar [-140.8, -38.4, -3, 140.8, 38.4, 1]
-
-data_augment:
- - NAME: random_world_flip
- ALONG_AXIS_LIST: [ 'x' ]
-
- - NAME: random_world_rotation
- WORLD_ROT_ANGLE: [ -0.78539816, 0.78539816 ]
-
- - NAME: random_world_scaling
- WORLD_SCALE_RANGE: [ 0.95, 1.05 ]
-
-# anchor box related
-postprocess:
- core_method: 'VoxelPostprocessor' # VoxelPostprocessor, BevPostprocessor supported
- anchor_args:
- cav_lidar_range: *cav_lidar
- l: 3.9
- w: 1.6
- h: 1.56
- r: [0, 90]
- feature_stride: 4
- num: &achor_num 2
- target_args:
- pos_threshold: 0.6
- neg_threshold: 0.45
- score_threshold: 0.20
- order: 'hwl' # hwl or lwh
- max_num: 100 # maximum number of objects in a single frame. use this number to make sure different frames has the same dimension in the same batch
- nms_thresh: 0.15
-
-# model related
-model:
- core_method: point_pillar_cobevt
- args:
- voxel_size: *voxel_size
- lidar_range: *cav_lidar
- anchor_number: *achor_num
- max_cav: *max_cav
- compression: 0 # compression rate
- backbone_fix: false
-
- pillar_vfe:
- use_norm: true
- with_distance: false
- use_absolute_xyz: true
- num_filters: [64]
- point_pillar_scatter:
- num_features: 64
-
- base_bev_backbone:
- layer_nums: [3, 5, 8]
- layer_strides: [2, 2, 2]
- num_filters: [64, 128, 256]
- upsample_strides: [1, 2, 4]
- num_upsample_filter: [128, 128, 128]
- shrink_header:
- kernal_size: [3]
- stride: [2]
- padding: [1]
- dim: [256]
- input_dim: 384 # 128 * 3
-
- fax_fusion:
- input_dim: 256
- mlp_dim: 256
- agent_size: *max_cav
- window_size: 4
- dim_head: 32
- drop_out: 0.1
- depth: 3
- mask: true
-
-
- # add decoder later
-
-loss:
- core_method: point_pillar_loss
- args:
- cls_weight: 1.0
- reg: 2.0
-
-optimizer:
- core_method: Adam
- lr: 0.001
- args:
- eps: 1e-10
- weight_decay: 1e-4
-
-lr_scheduler:
- core_method: cosineannealwarm #step, multistep, Exponential and cosineannealwarm support
- epoches: *epoches
- warmup_lr: 2e-4
- warmup_epoches: 10
- lr_min: 2e-5
diff --git a/OpenCOOD/opencood/hypes_yaml/point_pillar_early_fusion.yaml b/OpenCOOD/opencood/hypes_yaml/point_pillar_early_fusion.yaml
deleted file mode 100644
index 1c082dae..00000000
--- a/OpenCOOD/opencood/hypes_yaml/point_pillar_early_fusion.yaml
+++ /dev/null
@@ -1,96 +0,0 @@
-name: point_pillar_early_fusion
-root_dir: "opv2v_data_dumping/train"
-validate_dir: "opv2v_data_dumping/validate"
-
-yaml_parser: "load_point_pillar_params"
-train_params:
- batch_size: &batch_size 2
- epoches: 15
- eval_freq: 1
- save_freq: 1
-
-fusion:
- core_method: 'EarlyFusionDataset' # LateFusionDataset, EarlyFusionDataset, IntermediateFusionDataset supported
- args: []
-
-# preprocess-related
-preprocess:
- # options: BasePreprocessor, VoxelPreprocessor, BevPreprocessor
- core_method: 'SpVoxelPreprocessor'
- args:
- voxel_size: &voxel_size [0.4, 0.4, 4]
- max_points_per_voxel: 32
- max_voxel_train: 32000
- max_voxel_test: 70000
- # lidar range for each individual cav.
- cav_lidar_range: &cav_lidar [-140.8, -40, -3, 140.8, 40, 1]
-
-data_augment:
- - NAME: random_world_flip
- ALONG_AXIS_LIST: [ 'x' ]
-
- - NAME: random_world_rotation
- WORLD_ROT_ANGLE: [ -0.78539816, 0.78539816 ]
-
- - NAME: random_world_scaling
- WORLD_SCALE_RANGE: [ 0.95, 1.05 ]
-
-# anchor box related
-postprocess:
- core_method: 'VoxelPostprocessor' # VoxelPostprocessor, BevPostprocessor supported
- anchor_args:
- cav_lidar_range: *cav_lidar
- l: 3.9
- w: 1.6
- h: 1.56
- r: [0, 90]
- num: &achor_num 2
- target_args:
- pos_threshold: 0.6
- neg_threshold: 0.45
- score_threshold: 0.20
- order: 'hwl' # hwl or lwh
- max_num: 100 # maximum number of objects in a single frame. use this number to make sure different frames has the same dimension in the same batch
- nms_thresh: 0.15
-
-# model related
-model:
- core_method: point_pillar
- args:
- voxel_size: *voxel_size
- lidar_range: *cav_lidar
- anchor_number: *achor_num
- pillar_vfe:
- use_norm: true
- with_distance: false
- use_absolute_xyz: true
- num_filters: [64]
- point_pillar_scatter:
- num_features: 64
-
- base_bev_backbone:
- layer_nums: [3, 5, 8]
- layer_strides: [2, 2, 2]
- num_filters: [64, 128, 256]
- upsample_strides: [1, 2, 4]
- num_upsample_filter: [128, 128, 128]
-
- anchor_num: *achor_num
-
-loss:
- core_method: point_pillar_loss
- args:
- cls_weight: 1.0
- reg: 2.0
-
-optimizer:
- core_method: Adam
- lr: 0.002
- args:
- eps: 1e-10
- weight_decay: 1e-4
-
-lr_scheduler:
- core_method: multistep #step, multistep and exponential and cosineannealwarm support
- gamma: 0.1
- step_size: [10, 15]
diff --git a/OpenCOOD/opencood/hypes_yaml/point_pillar_fcooper.yaml b/OpenCOOD/opencood/hypes_yaml/point_pillar_fcooper.yaml
deleted file mode 100644
index 18ce02b3..00000000
--- a/OpenCOOD/opencood/hypes_yaml/point_pillar_fcooper.yaml
+++ /dev/null
@@ -1,106 +0,0 @@
-name: point_pillar_fcooper
-root_dir: 'opv2v_data_dumping/train'
-validate_dir: 'opv2v_data_dumping/validate'
-
-yaml_parser: "load_point_pillar_params"
-train_params:
- batch_size: &batch_size 2
- epoches: 15
- eval_freq: 1
- save_freq: 1
- max_cav: &max_cav 5
-
-fusion:
- core_method: 'IntermediateFusionDataset' # LateFusionDataset, EarlyFusionDataset, IntermediateFusionDataset supported
- args: []
-
-# preprocess-related
-preprocess:
- # options: BasePreprocessor, VoxelPreprocessor, BevPreprocessor
- core_method: 'SpVoxelPreprocessor'
- args:
- voxel_size: &voxel_size [0.4, 0.4, 4]
- max_points_per_voxel: 32
- max_voxel_train: 32000
- max_voxel_test: 70000
- # lidar range for each individual cav.
- cav_lidar_range: &cav_lidar [-140.8, -40, -3, 140.8, 40, 1]
-
-data_augment:
- - NAME: random_world_flip
- ALONG_AXIS_LIST: [ 'x' ]
-
- - NAME: random_world_rotation
- WORLD_ROT_ANGLE: [ -0.78539816, 0.78539816 ]
-
- - NAME: random_world_scaling
- WORLD_SCALE_RANGE: [ 0.95, 1.05 ]
-
-# anchor box related
-postprocess:
- core_method: 'VoxelPostprocessor' # VoxelPostprocessor, BevPostprocessor supported
- anchor_args:
- cav_lidar_range: *cav_lidar
- l: 3.9
- w: 1.6
- h: 1.56
- r: [0, 90]
- feature_stride: 2
- num: &achor_num 2
- target_args:
- pos_threshold: 0.6
- neg_threshold: 0.45
- score_threshold: 0.20
- order: 'hwl' # hwl or lwh
- max_num: 100 # maximum number of objects in a single frame. use this number to make sure different frames has the same dimension in the same batch
- nms_thresh: 0.15
-
-# model related
-model:
- core_method: point_pillar_fcooper
- args:
- voxel_size: *voxel_size
- lidar_range: *cav_lidar
- anchor_number: *achor_num
- max_cav: *max_cav
- compression: 0 # compression rate
- backbone_fix: false
-
- pillar_vfe:
- use_norm: true
- with_distance: false
- use_absolute_xyz: true
- num_filters: [64]
- point_pillar_scatter:
- num_features: 64
-
- base_bev_backbone:
- layer_nums: [3, 5, 8]
- layer_strides: [2, 2, 2]
- num_filters: [64, 128, 256]
- upsample_strides: [1, 2, 4]
- num_upsample_filter: [128, 128, 128]
- shrink_header:
- kernal_size: [ 1 ]
- stride: [ 1 ]
- padding: [ 0 ]
- dim: [ 256 ]
- input_dim: 384 # 128 * 3
-
-loss:
- core_method: point_pillar_loss
- args:
- cls_weight: 1.0
- reg: 2.0
-
-optimizer:
- core_method: Adam
- lr: 0.001
- args:
- eps: 1e-10
- weight_decay: 1e-4
-
-lr_scheduler:
- core_method: multistep #step, multistep and exponential and cosineannealwarm support
- gamma: 0.1
- step_size: [10, 15]
diff --git a/OpenCOOD/opencood/hypes_yaml/point_pillar_intermediate_V2VAM.yaml b/OpenCOOD/opencood/hypes_yaml/point_pillar_intermediate_V2VAM.yaml
deleted file mode 100644
index ee5f2426..00000000
--- a/OpenCOOD/opencood/hypes_yaml/point_pillar_intermediate_V2VAM.yaml
+++ /dev/null
@@ -1,107 +0,0 @@
-name: point_pillar_intermediate_V2VAM_nocompressionV2
-sub_name:
-root_dir: "opv2v_data_dumping/train"
-validate_dir: "opv2v_data_dumping/validate"
-
-yaml_parser: "load_point_pillar_params"
-train_params:
- batch_size: &batch_size 4
- epoches: 60
- eval_freq: 1
- save_freq: 1
- max_cav: &max_cav 5
-
-fusion:
- core_method: 'IntermediateFusionDataset' # LateFusionDataset, EarlyFusionDataset, IntermediateFusionDataset supported
- args: []
-
-# preprocess-related
-preprocess:
- # options: BasePreprocessor, VoxelPreprocessor, BevPreprocessor
- core_method: 'SpVoxelPreprocessor'
- args:
- voxel_size: &voxel_size [0.4, 0.4, 4]
- max_points_per_voxel: 32
- max_voxel_train: 32000
- max_voxel_test: 70000
- # lidar range for each individual cav.
- cav_lidar_range: &cav_lidar [-140.8, -40, -3, 140.8, 40, 1]
-
-data_augment:
- - NAME: random_world_flip
- ALONG_AXIS_LIST: [ 'x' ]
-
- - NAME: random_world_rotation
- WORLD_ROT_ANGLE: [ -0.78539816, 0.78539816 ]
-
- - NAME: random_world_scaling
- WORLD_SCALE_RANGE: [ 0.95, 1.05 ]
-
-# anchor box related
-postprocess:
- core_method: 'VoxelPostprocessor' # VoxelPostprocessor, BevPostprocessor supported
- anchor_args:
- cav_lidar_range: *cav_lidar
- l: 3.9
- w: 1.6
- h: 1.56
- r: [0, 90]
- feature_stride: 4
- num: &achor_num 2
- target_args:
- pos_threshold: 0.6
- neg_threshold: 0.45
- score_threshold: 0.20
- order: 'hwl' # hwl or lwh
- max_num: 100 # maximum number of objects in a single frame. use this number to make sure different frames has the same dimension in the same batch
- nms_thresh: 0.15
-
-# model related
-model:
- core_method: point_pillar_intermediate_V2VAM
- args:
- voxel_size: *voxel_size
- lidar_range: *cav_lidar
- anchor_number: *achor_num
- max_cav: *max_cav
- compression: 0 #32 # compression rate
- backbone_fix: false
-
- pillar_vfe:
- use_norm: true
- with_distance: false
- use_absolute_xyz: true
- num_filters: [64]
- point_pillar_scatter:
- num_features: 64
-
- base_bev_backbone:
- layer_nums: [3, 5, 8]
- layer_strides: [2, 2, 2]
- num_filters: [64, 128, 256]
- upsample_strides: [1, 2, 4]
- num_upsample_filter: [128, 128, 128]
- shrink_header:
- kernal_size: [3]
- stride: [2]
- padding: [1]
- dim: [256]
- input_dim: 384 # 128 * 3
-
-loss:
- core_method: point_pillar_loss
- args:
- cls_weight: 1.0
- reg: 2.0
-
-optimizer:
- core_method: Adam
- lr: 0.0000001
- args:
- eps: 1e-10
- weight_decay: 1e-4
-
-lr_scheduler:
- core_method: multistep #step, multistep and exponential and cosineannealwarm support
- gamma: 0.1
- step_size: [15,40,60]
diff --git a/OpenCOOD/opencood/hypes_yaml/point_pillar_intermediate_fusion.yaml b/OpenCOOD/opencood/hypes_yaml/point_pillar_intermediate_fusion.yaml
deleted file mode 100644
index b2084d37..00000000
--- a/OpenCOOD/opencood/hypes_yaml/point_pillar_intermediate_fusion.yaml
+++ /dev/null
@@ -1,99 +0,0 @@
-name: point_pillar_intermediate_fusion
-root_dir: "opv2v_data_dumping/train"
-validate_dir: "opv2v_data_dumping/validate"
-
-yaml_parser: "load_point_pillar_params"
-train_params:
- batch_size: &batch_size 2
- epoches: 15
- eval_freq: 1
- save_freq: 1
-
-fusion:
- core_method: 'IntermediateFusionDataset' # LateFusionDataset, EarlyFusionDataset, IntermediateFusionDataset supported
- args: []
-
-# preprocess-related
-preprocess:
- # options: BasePreprocessor, VoxelPreprocessor, BevPreprocessor
- core_method: 'SpVoxelPreprocessor'
- args:
- voxel_size: &voxel_size [0.4, 0.4, 4]
- max_points_per_voxel: 32
- max_voxel_train: 32000
- max_voxel_test: 70000
- # lidar range for each individual cav.
- cav_lidar_range: &cav_lidar [-140.8, -40, -3, 140.8, 40, 1]
-
-data_augment:
- - NAME: random_world_flip
- ALONG_AXIS_LIST: [ 'x' ]
-
- - NAME: random_world_rotation
- WORLD_ROT_ANGLE: [ -0.78539816, 0.78539816 ]
-
- - NAME: random_world_scaling
- WORLD_SCALE_RANGE: [ 0.95, 1.05 ]
-
-# anchor box related
-postprocess:
- core_method: 'VoxelPostprocessor' # VoxelPostprocessor, BevPostprocessor supported
- anchor_args:
- cav_lidar_range: *cav_lidar
- l: 3.9
- w: 1.6
- h: 1.56
- r: [0, 90]
- feature_stride: 2
- num: &achor_num 2
- target_args:
- pos_threshold: 0.6
- neg_threshold: 0.45
- score_threshold: 0.20
- order: 'hwl' # hwl or lwh
- max_num: 100 # maximum number of objects in a single frame. use this number to make sure different frames has the same dimension in the same batch
- nms_thresh: 0.15
-
-# model related
-model:
- core_method: point_pillar_intermediate
- args:
- voxel_size: *voxel_size
- lidar_range: *cav_lidar
- anchor_number: *achor_num
-
- pillar_vfe:
- use_norm: true
- with_distance: false
- use_absolute_xyz: true
- num_filters: [64]
- point_pillar_scatter:
- num_features: 64
-
- base_bev_backbone:
- layer_nums: [3, 5, 8]
- layer_strides: [2, 2, 2]
- num_filters: [64, 128, 256]
- upsample_strides: [1, 2, 4]
- num_upsample_filter: [128, 128, 128]
- compression: 0
-
- anchor_num: *achor_num
-
-loss:
- core_method: point_pillar_loss
- args:
- cls_weight: 1.0
- reg: 2.0
-
-optimizer:
- core_method: Adam
- lr: 0.002
- args:
- eps: 1e-10
- weight_decay: 1e-4
-
-lr_scheduler:
- core_method: multistep #step, multistep and exponential and cosineannealwarm support
- gamma: 0.1
- step_size: [10, 15]
diff --git a/OpenCOOD/opencood/hypes_yaml/point_pillar_late_fusion.yaml b/OpenCOOD/opencood/hypes_yaml/point_pillar_late_fusion.yaml
deleted file mode 100644
index 26cb8ad5..00000000
--- a/OpenCOOD/opencood/hypes_yaml/point_pillar_late_fusion.yaml
+++ /dev/null
@@ -1,96 +0,0 @@
-name: point_pillar_late_fusion
-root_dir: "opv2v_data_dumping/train"
-validate_dir: "opv2v_data_dumping/validate"
-
-yaml_parser: "load_point_pillar_params"
-train_params:
- batch_size: &batch_size 2
- epoches: 15
- eval_freq: 1
- save_freq: 1
-
-fusion:
- core_method: 'LateFusionDataset' # LateFusionDataset, EarlyFusionDataset, IntermediateFusionDataset supported
- args: []
-
-# preprocess-related
-preprocess:
- # options: BasePreprocessor, VoxelPreprocessor, BevPreprocessor
- core_method: 'SpVoxelPreprocessor'
- args:
- voxel_size: &voxel_size [0.4, 0.4, 4]
- max_points_per_voxel: 32
- max_voxel_train: 16000
- max_voxel_test: 40000
- # lidar range for each individual cav.
- cav_lidar_range: &cav_lidar [-70.4, -40, -3, 70.4, 40, 1]
-
-data_augment:
- - NAME: random_world_flip
- ALONG_AXIS_LIST: [ 'x' ]
-
- - NAME: random_world_rotation
- WORLD_ROT_ANGLE: [ -0.78539816, 0.78539816 ]
-
- - NAME: random_world_scaling
- WORLD_SCALE_RANGE: [ 0.95, 1.05 ]
-
-# anchor box related
-postprocess:
- core_method: 'VoxelPostprocessor' # VoxelPostprocessor, BevPostprocessor supported
- anchor_args:
- cav_lidar_range: *cav_lidar
- l: 3.9
- w: 1.6
- h: 1.56
- r: [0, 90]
- num: &achor_num 2
- target_args:
- pos_threshold: 0.6
- neg_threshold: 0.45
- score_threshold: 0.20
- order: 'hwl' # hwl or lwh
- max_num: 100 # maximum number of objects in a single frame. use this number to make sure different frames has the same dimension in the same batch
- nms_thresh: 0.15
-
-# model related
-model:
- core_method: point_pillar
- args:
- voxel_size: *voxel_size
- lidar_range: *cav_lidar
- anchor_number: *achor_num
- pillar_vfe:
- use_norm: true
- with_distance: false
- use_absolute_xyz: true
- num_filters: [64]
- point_pillar_scatter:
- num_features: 64
-
- base_bev_backbone:
- layer_nums: [3, 5, 8]
- layer_strides: [2, 2, 2]
- num_filters: [64, 128, 256]
- upsample_strides: [1, 2, 4]
- num_upsample_filter: [128, 128, 128]
-
- anchor_num: *achor_num
-
-loss:
- core_method: point_pillar_loss
- args:
- cls_weight: 1.0
- reg: 2.0
-
-optimizer:
- core_method: Adam
- lr: 0.002
- args:
- eps: 1e-10
- weight_decay: 1e-4
-
-lr_scheduler:
- core_method: multistep #step, multistep and exponential and cosineannealwarm support
- gamma: 0.1
- step_size: [10, 15]
diff --git a/OpenCOOD/opencood/hypes_yaml/point_pillar_v2vnet.yaml b/OpenCOOD/opencood/hypes_yaml/point_pillar_v2vnet.yaml
deleted file mode 100644
index dc623570..00000000
--- a/OpenCOOD/opencood/hypes_yaml/point_pillar_v2vnet.yaml
+++ /dev/null
@@ -1,123 +0,0 @@
-name: point_pillar_v2vnet
-root_dir: 'opv2v_data_dumping/train'
-validate_dir: 'opv2v_data_dumping/validate'
-
-yaml_parser: "load_point_pillar_params"
-train_params:
- batch_size: &batch_size 4
- epoches: 60
- eval_freq: 1
- save_freq: 1
- max_cav: &max_cav 5
-
-fusion:
- core_method: 'IntermediateFusionDataset' # LateFusionDataset, EarlyFusionDataset, IntermediateFusionDataset supported
- args:
- proj_first: true
-
-# preprocess-related
-preprocess:
- # options: BasePreprocessor, VoxelPreprocessor, BevPreprocessor
- core_method: 'SpVoxelPreprocessor'
- args:
- voxel_size: &voxel_size [0.4, 0.4, 4]
- max_points_per_voxel: 32
- max_voxel_train: 32000
- max_voxel_test: 70000
- # lidar range for each individual cav.
- cav_lidar_range: &cav_lidar [-140.8, -40, -3, 140.8, 40, 1]
-
-data_augment:
- - NAME: random_world_flip
- ALONG_AXIS_LIST: [ 'x' ]
-
- - NAME: random_world_rotation
- WORLD_ROT_ANGLE: [ -0.78539816, 0.78539816 ]
-
- - NAME: random_world_scaling
- WORLD_SCALE_RANGE: [ 0.95, 1.05 ]
-
-# anchor box related
-postprocess:
- core_method: 'VoxelPostprocessor' # VoxelPostprocessor, BevPostprocessor supported
- anchor_args:
- cav_lidar_range: *cav_lidar
- l: 3.9
- w: 1.6
- h: 1.56
- r: [0, 90]
- feature_stride: 4
- num: &achor_num 2
- target_args:
- pos_threshold: 0.6
- neg_threshold: 0.45
- score_threshold: 0.20
- order: 'hwl' # hwl or lwh
- max_num: 100 # maximum number of objects in a single frame. use this number to make sure different frames has the same dimension in the same batch
- nms_thresh: 0.15
-
-# model related
-model:
- core_method: point_pillar_v2vnet
- args:
- voxel_size: *voxel_size
- lidar_range: *cav_lidar
- anchor_number: *achor_num
- max_cav: *max_cav
- compression: 0 # compression rate
- backbone_fix: false
-
- pillar_vfe:
- use_norm: true
- with_distance: false
- use_absolute_xyz: true
- num_filters: [64]
- point_pillar_scatter:
- num_features: 64
-
- base_bev_backbone:
- layer_nums: [3, 5, 8]
- layer_strides: [2, 2, 2]
- num_filters: [64, 128, 256]
- upsample_strides: [1, 2, 4]
- num_upsample_filter: [128, 128, 128]
- shrink_header:
- kernal_size: [ 3 ]
- stride: [ 2 ]
- padding: [ 1 ]
- dim: [ 256 ]
- input_dim: 384 # 128 * 3
-
- v2vfusion:
- voxel_size: *voxel_size
- downsample_rate: 4
- num_iteration: 2
- in_channels: 256
- gru_flag: true
- agg_operator: "avg" # max or avg
- conv_gru:
- H: 50
- W: 176
- num_layers: 1
- kernel_size: [[3,3]]
-
-
- # add decoder later
-
-loss:
- core_method: point_pillar_loss
- args:
- cls_weight: 1.0
- reg: 2.0
-
-optimizer:
- core_method: Adam
- lr: 0.001
- args:
- eps: 1e-10
- weight_decay: 1e-4
-
-lr_scheduler:
- core_method: multistep #step, multistep and exponential and cosineannealwarm support
- gamma: 0.1
- step_size: [10, 50, 80]
diff --git a/OpenCOOD/opencood/hypes_yaml/point_pillar_v2xvit.yaml b/OpenCOOD/opencood/hypes_yaml/point_pillar_v2xvit.yaml
deleted file mode 100644
index 07765327..00000000
--- a/OpenCOOD/opencood/hypes_yaml/point_pillar_v2xvit.yaml
+++ /dev/null
@@ -1,161 +0,0 @@
-name: point_pillar_v2xvit
-root_dir: 'v2xset/train'
-validate_dir: 'v2xset/validate'
-
-wild_setting:
- async: false
- async_mode: 'sim'
- async_overhead: 100
- seed: 25
- loc_err: false
- xyz_std: 0.2
- ryp_std: 0.2
- data_size: 1.06 # Mb!!
- transmission_speed: 27 # Mbps!!
- backbone_delay: 10 # ms
-
-yaml_parser: "load_point_pillar_params"
-train_params:
- batch_size: &batch_size 2
- epoches: 60
- eval_freq: 1
- save_freq: 1
- max_cav: &max_cav 5
-
-fusion:
- core_method: 'IntermediateFusionDataset' # LateFusionDataset, EarlyFusionDataset, IntermediateFusionDataset supported
- args:
- cur_ego_pose_flag: False
- # when the cur_ego_pose_flag is set to True, there is no time gap
- # between the time when the LiDAR data is captured by connected
- # agents and when the extracted features are received by
- # the ego vehicle, which is equal to implement STCM. When set to False,
- # STCM has to be used.
-
-# preprocess-related
-preprocess:
- # options: BasePreprocessor, VoxelPreprocessor, BevPreprocessor
- core_method: 'SpVoxelPreprocessor'
- args:
- voxel_size: &voxel_size [0.4, 0.4, 4]
- max_points_per_voxel: 32
- max_voxel_train: 32000
- max_voxel_test: 70000
- # lidar range for each individual cav.
- cav_lidar_range: &cav_lidar [-140.8, -38.4, -3, 140.8, 38.4, 1]
-
-data_augment:
- - NAME: random_world_flip
- ALONG_AXIS_LIST: [ 'x' ]
-
- - NAME: random_world_rotation
- WORLD_ROT_ANGLE: [ -0.78539816, 0.78539816 ]
-
- - NAME: random_world_scaling
- WORLD_SCALE_RANGE: [ 0.95, 1.05 ]
-
-# anchor box related
-postprocess:
- core_method: 'VoxelPostprocessor' # VoxelPostprocessor, BevPostprocessor supported
- anchor_args:
- cav_lidar_range: *cav_lidar
- l: 3.9
- w: 1.6
- h: 1.56
- r: [0, 90]
- feature_stride: 4
- num: &achor_num 2
- target_args:
- pos_threshold: 0.6
- neg_threshold: 0.45
- score_threshold: 0.27
- order: 'hwl' # hwl or lwh
- max_num: 100 # maximum number of objects in a single frame. use this number to make sure different frames has the same dimension in the same batch
- nms_thresh: 0.15
-
-# model related
-model:
- core_method: point_pillar_transformer
- args:
- voxel_size: *voxel_size
- lidar_range: *cav_lidar
- anchor_number: *achor_num
- max_cav: *max_cav
- compression: 0 # compression rate
- backbone_fix: false
-
- pillar_vfe:
- use_norm: true
- with_distance: false
- use_absolute_xyz: true
- num_filters: [64]
- point_pillar_scatter:
- num_features: 64
-
- base_bev_backbone:
- layer_nums: [3, 5, 8]
- layer_strides: [2, 2, 2]
- num_filters: [64, 128, 256]
- upsample_strides: [1, 2, 4]
- num_upsample_filter: [128, 128, 128]
- shrink_header:
- kernal_size: [3]
- stride: [2]
- padding: [1]
- dim: [256]
- input_dim: 384 # 128 * 3
-
- transformer:
- encoder: &encoder
- # number of fusion blocks per encoder layer
- num_blocks: 1
- # number of encoder layers
- depth: 3
- use_roi_mask: true
- use_RTE: &use_RTE true
- RTE_ratio: &RTE_ratio 2 # 2 means the dt has 100ms interval while 1 means 50 ms interval
- # agent-wise attention
- cav_att_config: &cav_att_config
- dim: 256
- use_hetero: true
- use_RTE: *use_RTE
- RTE_ratio: *RTE_ratio
- heads: 8
- dim_head: 32
- dropout: 0.3
- # spatial-wise attention
- pwindow_att_config: &pwindow_att_config
- dim: 256
- heads: [16, 8, 4]
- dim_head: [16, 32, 64]
- dropout: 0.3
- window_size: [4, 8, 16]
- relative_pos_embedding: true
- fusion_method: 'split_attn'
- # feedforward condition
- feed_forward: &feed_forward
- mlp_dim: 256
- dropout: 0.3
- sttf: &sttf
- voxel_size: *voxel_size
- downsample_rate: 4
-
- # add decoder later
-
-loss:
- core_method: point_pillar_loss
- args:
- cls_weight: 1.0
- reg: 2.0
-
-optimizer:
- core_method: Adam
- lr: 0.001
- args:
- eps: 1e-10
- weight_decay: 1e-4
-
-lr_scheduler:
- core_method: multistep #step, multistep and exponential and cosineannealwarm support
- gamma: 0.1
- step_size: [15, 50]
diff --git a/OpenCOOD/opencood/hypes_yaml/point_pillar_where2comm.yaml b/OpenCOOD/opencood/hypes_yaml/point_pillar_where2comm.yaml
deleted file mode 100644
index 93966ad4..00000000
--- a/OpenCOOD/opencood/hypes_yaml/point_pillar_where2comm.yaml
+++ /dev/null
@@ -1,133 +0,0 @@
-name: point_pillar_where2comm
-root_dir: "opv2v_data_dumping/train"
-validate_dir: "opv2v_data_dumping/validate"
-
-
-
-wild_setting:
- seed: 20
- async: False
- async_mode: 'sim'
- async_overhead: 100
- loc_err: False
- xyz_std: 0.2
- ryp_std: 0.2
- data_size: 1.06 # Mb
- transmission_speed: 27 # Mbps
- backbone_delay: 10 # ms
-
-yaml_parser: 'load_point_pillar_params'
-train_params:
- batch_size: &batch_size 4
- epoches: &epoches 100
- eval_freq: 1
- save_freq: 1
- max_cav: &max_cav 8
-
-fusion:
- core_method: 'IntermediateFusionDataset' # LateFusionDataset, EarlyFusionDataset, IntermediateFusionDataset supported
- args: [ ]
-
-# Preprocess-related
-preprocess:
- # Options: BasePreprocessor, VoxelPreprocessor, BevPreprocessor
- core_method: 'SpVoxelPreprocessor'
- args:
- voxel_size: &voxel_size [ 0.4, 0.4, 16 ]
- max_points_per_voxel: 32
- max_voxel_train: 32000
- max_voxel_test: 70000
- # LiDAR range for each individual CAV
- cav_lidar_range: &cav_lidar [ -140.8, -38.4, -8 , 140.8, 38.4, 8 ]
-
-data_augment:
- - NAME: random_world_flip
- ALONG_AXIS_LIST: [ 'x' ]
- - NAME: random_world_rotation
- WORLD_ROT_ANGLE: [ -0.78539816, 0.78539816 ]
- - NAME: random_world_scaling
- WORLD_SCALE_RANGE: [ 0.95, 1.05 ]
-
-# Anchor box related
-postprocess:
- core_method: 'VoxelPostprocessor' # VoxelPostprocessor, BevPostprocessor supported
- anchor_args:
- cav_lidar_range: *cav_lidar
- l: 3.9
- w: 1.6
- h: 1.56
- r: [ 0, 90 ]
- num: &achor_num 2
- feature_stride: 4
- target_args:
- pos_threshold: 0.6
- neg_threshold: 0.45
- score_threshold: 0.2
- order: 'hwl' # hwl or lwh
- max_num: 100 # Maximum number of objects in a single frame. Use this number to make sure different frames has the same dimension in the same batch
- nms_thresh: 0.15
-
-# Model related
-model:
- core_method: point_pillar_where2comm
- args:
- voxel_size: *voxel_size
- lidar_range: *cav_lidar
- anchor_number: *achor_num
- head_dim: 256
- max_cav: *max_cav
- compression: 0 # Compression rate
- backbone_fix: False
- pillar_vfe:
- use_norm: True
- with_distance: False
- use_absolute_xyz: True
- num_filters: [ 64 ]
- point_pillar_scatter:
- num_features: 64
- base_bev_backbone:
- layer_nums: &layer_nums [ 3, 5, 8 ]
- layer_strides: [ 2, 2, 2 ]
- num_filters: &num_filters [ 64, 128, 256 ]
- upsample_strides: [ 1, 2, 4 ]
- num_upsample_filter: [ 128, 128, 128 ]
- shrink_header:
- kernal_size: [ 3 ]
- stride: [ 2 ]
- padding: [ 1 ]
- dim: [ 256 ]
- input_dim: 384 # 128 * 3
- where2comm_fusion:
- fully: False
- voxel_size: *voxel_size
- downsample_rate: 4
- in_channels: 256
- multi_scale: True
- layer_nums: *layer_nums
- num_filters: *num_filters
- communication:
- round: 1
- threshold: 0.01
- gaussian_smooth:
- k_size: 5
- c_sigma: 1.0
-
-loss:
- core_method: point_pillar_loss
- args:
- cls_weight: 1.0
- reg: 2.0
-
-optimizer:
- core_method: Adam
- lr: 0.005
- args:
- eps: 1e-10
- weight_decay: 1e-2
-
-lr_scheduler:
- core_method: cosineannealwarm # step, multistep, exponential and cosineannealwarm support
- epoches: *epoches
- warmup_lr: 0.005
- warmup_epoches: 10
- lr_min: 5e-6
diff --git a/OpenCOOD/opencood/hypes_yaml/second_early_fusion.yaml b/OpenCOOD/opencood/hypes_yaml/second_early_fusion.yaml
deleted file mode 100644
index abf01980..00000000
--- a/OpenCOOD/opencood/hypes_yaml/second_early_fusion.yaml
+++ /dev/null
@@ -1,98 +0,0 @@
-name: second_early_fusion
-root_dir: "opv2v_data_dumping/train"
-validate_dir: "opv2v_data_dumping/validate"
-
-yaml_parser: "load_second_params"
-train_params:
- batch_size: &batch_size 2
- epoches: 30
- eval_freq: 1
- save_freq: 1
-
-fusion:
- core_method: 'EarlyFusionDataset' # LateFusionDataset, EarlyFusionDataset, IntermediateFusionDataset supported
- args: []
-
-# preprocess-related
-preprocess:
- # options: BasePreprocessor, VoxelPreprocessor, BevPreprocessor
- core_method: 'SpVoxelPreprocessor'
- args:
- voxel_size: &voxel_size [0.1, 0.1, 0.1]
- max_points_per_voxel: 5
- max_voxel_train: 32000
- max_voxel_test: 70000
- # lidar range for each individual cav.
- cav_lidar_range: &cav_lidar [-140.8, -40, -3, 140.8, 40, 1]
-
-data_augment:
- - NAME: random_world_flip
- ALONG_AXIS_LIST: [ 'x' ]
-
- - NAME: random_world_rotation
- WORLD_ROT_ANGLE: [ -0.78539816, 0.78539816 ]
-
- - NAME: random_world_scaling
- WORLD_SCALE_RANGE: [ 0.95, 1.05 ]
-
-# anchor box related
-postprocess:
- core_method: 'VoxelPostprocessor' # VoxelPostprocessor, BevPostprocessor supported
- anchor_args:
- cav_lidar_range: *cav_lidar
- l: 3.9
- w: 1.6
- h: 1.56
- r: [0, 90]
- feature_stride: 8
- num: &achor_num 2
- target_args:
- pos_threshold: 0.6
- neg_threshold: 0.45
- score_threshold: 0.20
- order: 'hwl' # hwl or lwh
- max_num: 100 # maximum number of objects in a single frame. use this number to make sure different frames has the same dimension in the same batch
- nms_thresh: 0.15
-
-# model related
-model:
- core_method: second
- args:
- batch_size: *batch_size
- voxel_size: *voxel_size
- lidar_range: *cav_lidar
- anchor_number: *achor_num
- mean_vfe:
- feature_points: 4
-
- backbone_3d: []
-
- height_compression:
- feature_num: 256
-
- base_bev_backbone:
- layer_nums: [5, 5]
- layer_strides: [1, 2]
- num_filters: [128, 256]
- upsample_strides: [1, 2]
- num_upsample_filter: [256, 256]
-
- anchor_num: *achor_num
-
-loss:
- core_method: point_pillar_loss
- args:
- cls_weight: 1.0
- reg: 2.0
-
-optimizer:
- core_method: Adam
- lr: 0.002
- args:
- eps: 1e-10
- weight_decay: 1e-4
-
-lr_scheduler:
- core_method: multistep #step, multistep and exponential and cosineannealwarm support
- gamma: 0.1
- step_size: [15, 30]
diff --git a/OpenCOOD/opencood/hypes_yaml/second_intermediate_fusion.yaml b/OpenCOOD/opencood/hypes_yaml/second_intermediate_fusion.yaml
deleted file mode 100644
index 9e29b595..00000000
--- a/OpenCOOD/opencood/hypes_yaml/second_intermediate_fusion.yaml
+++ /dev/null
@@ -1,98 +0,0 @@
-name: second_intermediate_fusion
-root_dir: "opv2v_data_dumping/train"
-validate_dir: "opv2v_data_dumping/validate"
-
-yaml_parser: "load_second_params"
-train_params:
- batch_size: &batch_size 2
- epoches: 60
- eval_freq: 1
- save_freq: 1
-
-fusion:
- core_method: 'IntermediateFusionDataset' # LateFusionDataset, EarlyFusionDataset, IntermediateFusionDataset supported
- args: []
-
-# preprocess-related
-preprocess:
- # options: BasePreprocessor, VoxelPreprocessor, BevPreprocessor
- core_method: 'SpVoxelPreprocessor'
- args:
- voxel_size: &voxel_size [0.1, 0.1, 0.1]
- max_points_per_voxel: 5
- max_voxel_train: 36000
- max_voxel_test: 70000
- # lidar range for each individual cav.
- cav_lidar_range: &cav_lidar [-140.8, -40, -3, 140.8, 40, 1]
-
-data_augment:
- - NAME: random_world_flip
- ALONG_AXIS_LIST: [ 'x' ]
-
- - NAME: random_world_rotation
- WORLD_ROT_ANGLE: [ -0.78539816, 0.78539816 ]
-
- - NAME: random_world_scaling
- WORLD_SCALE_RANGE: [ 0.95, 1.05 ]
-
-# anchor box related
-postprocess:
- core_method: 'VoxelPostprocessor' # VoxelPostprocessor, BevPostprocessor supported
- anchor_args:
- cav_lidar_range: *cav_lidar
- l: 3.9
- w: 1.6
- h: 1.56
- r: [0, 90]
- feature_stride: 8
- num: &achor_num 2
- target_args:
- pos_threshold: 0.6
- neg_threshold: 0.45
- score_threshold: 0.20
- order: 'hwl' # hwl or lwh
- max_num: 100 # maximum number of objects in a single frame. use this number to make sure different frames has the same dimension in the same batch
- nms_thresh: 0.15
-
-# model related
-model:
- core_method: second_intermediate
- args:
- batch_size: *batch_size
- voxel_size: *voxel_size
- lidar_range: *cav_lidar
- anchor_number: *achor_num
- mean_vfe:
- feature_points: 4
-
- backbone_3d: []
-
- height_compression:
- feature_num: 256
-
- base_bev_backbone:
- layer_nums: [5, 5]
- layer_strides: [1, 2]
- num_filters: [128, 256]
- upsample_strides: [1, 2]
- num_upsample_filter: [256, 256]
-
- anchor_num: *achor_num
-
-loss:
- core_method: point_pillar_loss
- args:
- cls_weight: 1.0
- reg: 2.0
-
-optimizer:
- core_method: Adam
- lr: 0.002
- args:
- eps: 1e-10
- weight_decay: 1e-4
-
-lr_scheduler:
- core_method: multistep #step, multistep and exponential and cosineannealwarm support
- gamma: 0.1
- step_size: [15, 30]
diff --git a/OpenCOOD/opencood/hypes_yaml/second_late_fusion.yaml b/OpenCOOD/opencood/hypes_yaml/second_late_fusion.yaml
deleted file mode 100644
index 8e72919a..00000000
--- a/OpenCOOD/opencood/hypes_yaml/second_late_fusion.yaml
+++ /dev/null
@@ -1,98 +0,0 @@
-name: second_late_fusion_low_res
-root_dir: "opv2v_data_dumping/train"
-validate_dir: "opv2v_data_dumping/validate"
-
-yaml_parser: "load_second_params"
-train_params:
- batch_size: &batch_size 2
- epoches: 40
- eval_freq: 1
- save_freq: 1
-
-fusion:
- core_method: 'LateFusionDataset' # LateFusionDataset, EarlyFusionDataset, IntermediateFusionDataset supported
- args: []
-
-# preprocess-related
-preprocess:
- # options: BasePreprocessor, VoxelPreprocessor, BevPreprocessor
- core_method: 'SpVoxelPreprocessor'
- args:
- voxel_size: &voxel_size [0.1, 0.1, 0.1]
- max_points_per_voxel: 5
- max_voxel_train: 16000
- max_voxel_test: 40000
- # lidar range for each individual cav.
- cav_lidar_range: &cav_lidar [-70.4, -40, -3, 70.4, 40, 1]
-
-data_augment:
- - NAME: random_world_flip
- ALONG_AXIS_LIST: [ 'x' ]
-
- - NAME: random_world_rotation
- WORLD_ROT_ANGLE: [ -0.78539816, 0.78539816 ]
-
- - NAME: random_world_scaling
- WORLD_SCALE_RANGE: [ 0.95, 1.05 ]
-
-# anchor box related
-postprocess:
- core_method: 'VoxelPostprocessor' # VoxelPostprocessor, BevPostprocessor supported
- anchor_args:
- cav_lidar_range: *cav_lidar
- l: 3.9
- w: 1.6
- h: 1.56
- r: [0, 90]
- feature_stride: 8
- num: &achor_num 2
- target_args:
- pos_threshold: 0.6
- neg_threshold: 0.45
- score_threshold: 0.20
- order: 'hwl' # hwl or lwh
- max_num: 100 # maximum number of objects in a single frame. use this number to make sure different frames has the same dimension in the same batch
- nms_thresh: 0.15
-
-# model related
-model:
- core_method: second
- args:
- batch_size: *batch_size
- voxel_size: *voxel_size
- lidar_range: *cav_lidar
- anchor_number: *achor_num
- mean_vfe:
- feature_points: 4
-
- backbone_3d: []
-
- height_compression:
- feature_num: 256
-
- base_bev_backbone:
- layer_nums: [5, 5]
- layer_strides: [1, 2]
- num_filters: [128, 256]
- upsample_strides: [1, 2]
- num_upsample_filter: [256, 256]
-
- anchor_num: *achor_num
-
-loss:
- core_method: point_pillar_loss
- args:
- cls_weight: 1.0
- reg: 2.0
-
-optimizer:
- core_method: Adam
- lr: 0.002
- args:
- eps: 1e-10
- weight_decay: 1e-4
-
-lr_scheduler:
- core_method: multistep #step, multistep and exponential and cosineannealwarm support
- gamma: 0.1
- step_size: [15, 30]
diff --git a/OpenCOOD/opencood/hypes_yaml/visualization.yaml b/OpenCOOD/opencood/hypes_yaml/visualization.yaml
deleted file mode 100644
index 15853ebb..00000000
--- a/OpenCOOD/opencood/hypes_yaml/visualization.yaml
+++ /dev/null
@@ -1,56 +0,0 @@
-# this yaml is only for visualization
-name: visualization
-
-yaml_parser: "load_voxel_params"
-root_dir: "opv2v_data_dumping/train"
-validate_dir: "opv2v_data_dumping/test"
-
-train_params:
- batch_size: &batch_size 4
- epoches: 100
- eval_freq: 1
- save_freq: 1
-
-fusion:
- core_method: 'EarlyFusionDataset' # LateFusionDataset, EarlyFusionDataset, IntermediateFusionDataset supported
- args: []
-
-# preprocess-related
-preprocess:
- # options: BasePreprocessor, VoxelPreprocessor, BevPreprocessor
- core_method: 'SpVoxelPreprocessor'
- args:
- voxel_size: &voxel_size [0.4, 0.4, 0.4]
- max_points_per_voxel: &T 32
- max_voxel_train: 36000
- max_voxel_test: 70000
- # lidar range for each individual cav.
- cav_lidar_range: &cav_lidar [-140.8, -40, -3, 140.8, 40, 1]
-
-data_augment:
- - NAME: random_world_flip
- ALONG_AXIS_LIST: [ 'x' ]
-
- - NAME: random_world_rotation
- WORLD_ROT_ANGLE: [ -0.78539816, 0.78539816 ]
-
- - NAME: random_world_scaling
- WORLD_SCALE_RANGE: [ 0.95, 1.05 ]
-
-# anchor box related
-postprocess:
- core_method: 'VoxelPostprocessor' # VoxelPostprocessor, BevPostprocessor supported
- anchor_args:
- cav_lidar_range: *cav_lidar
- l: 3.9
- w: 1.6
- h: 1.56
- r: [0, 90]
- num: &achor_num 2
- target_args:
- pos_threshold: 0.6
- neg_threshold: 0.45
- score_threshold: 0.96
- order: 'hwl' # hwl or lwh
- max_num: 100 # maximum number of objects in a single frame. use this number to make sure different frames has the same dimension in the same batch
- nms_thresh: 0.15
diff --git a/OpenCOOD/opencood/hypes_yaml/voxelnet_early_fusion.yaml b/OpenCOOD/opencood/hypes_yaml/voxelnet_early_fusion.yaml
deleted file mode 100644
index ca958525..00000000
--- a/OpenCOOD/opencood/hypes_yaml/voxelnet_early_fusion.yaml
+++ /dev/null
@@ -1,88 +0,0 @@
-name: voxelnet_early_fusion
-
-yaml_parser: "load_voxel_params"
-root_dir: "opv2v_data_dumping/train"
-validate_dir: "opv2v_data_dumping/validate"
-
-train_params:
- batch_size: &batch_size 4
- epoches: 100
- eval_freq: 1
- save_freq: 1
-
-fusion:
- core_method: 'EarlyFusionDataset' # LateFusionDataset, EarlyFusionDataset, IntermediateFusionDataset supported
- args: []
-
-# preprocess-related
-preprocess:
- # options: BasePreprocessor, VoxelPreprocessor, BevPreprocessor
- core_method: 'SpVoxelPreprocessor'
- args:
- voxel_size: &voxel_size [0.4, 0.4, 0.4]
- max_points_per_voxel: &T 32
- max_voxel_train: 36000
- max_voxel_test: 70000
- # lidar range for each individual cav.
- cav_lidar_range: &cav_lidar [-140.8, -40, -3, 140.8, 40, 1]
-
-data_augment:
- - NAME: random_world_flip
- ALONG_AXIS_LIST: [ 'x' ]
-
- - NAME: random_world_rotation
- WORLD_ROT_ANGLE: [ -0.78539816, 0.78539816 ]
-
- - NAME: random_world_scaling
- WORLD_SCALE_RANGE: [ 0.95, 1.05 ]
-
-# anchor box related
-postprocess:
- core_method: 'VoxelPostprocessor' # VoxelPostprocessor, BevPostprocessor supported
- anchor_args:
- cav_lidar_range: *cav_lidar
- l: 3.9
- w: 1.6
- h: 1.56
- r: [0, 90]
- num: &achor_num 2
- target_args:
- pos_threshold: 0.6
- neg_threshold: 0.45
- score_threshold: 0.96
- order: 'hwl' # hwl or lwh
- max_num: 100 # maximum number of objects in a single frame. use this number to make sure different frames has the same dimension in the same batch
- nms_thresh: 0.20
-
-# model related
-model:
- core_method: voxel_net # corresponding to VoxelNet
- args:
- N: *batch_size
- T: *T
- lidar_range: *cav_lidar
- voxel_size: *voxel_size
- anchor_num: *achor_num
- pillar_vfe:
- use_norm: true
- with_distance: false
- use_absolute_xyz: true
- num_filters: [ 64 ]
-
-loss:
- core_method: point_pillar_loss
- args:
- cls_weight: 1.0
- reg: 2.0
-
-optimizer:
- core_method: Adam
- lr: 0.002
- args:
- eps: 1e-10
- weight_decay: 1e-4
-
-lr_scheduler:
- core_method: multistep #step, multistep and exponential and cosineannealwarm support
- gamma: 0.1
- step_size: [15, 30]
diff --git a/OpenCOOD/opencood/hypes_yaml/voxelnet_intermediate_fusion.yaml b/OpenCOOD/opencood/hypes_yaml/voxelnet_intermediate_fusion.yaml
deleted file mode 100644
index 4630f521..00000000
--- a/OpenCOOD/opencood/hypes_yaml/voxelnet_intermediate_fusion.yaml
+++ /dev/null
@@ -1,89 +0,0 @@
-name: voxelnet_intermediate_fusion
-
-yaml_parser: "load_voxel_params"
-root_dir: "opv2v_data_dumping/train"
-validate_dir: "opv2v_data_dumping/validate"
-
-train_params:
- batch_size: &batch_size 4
- epoches: 100
- eval_freq: 1
- save_freq: 1
-
-fusion:
- core_method: 'IntermediateFusionDataset' # LateFusionDataset, EarlyFusionDataset, IntermediateFusionDataset supported
- args: []
-
-# preprocess-related
-preprocess:
- # options: BasePreprocessor, VoxelPreprocessor, BevPreprocessor
- core_method: 'SpVoxelPreprocessor'
- args:
- voxel_size: &voxel_size [0.4, 0.4, 0.4]
- max_points_per_voxel: &T 32
- max_voxel_train: 36000
- max_voxel_test: 70000
- # lidar range for each individual cav.
- cav_lidar_range: &cav_lidar [-140.8, -41.6, -8, 140.8, 41.6, 8]
-
-data_augment:
- - NAME: random_world_flip
- ALONG_AXIS_LIST: [ 'x' ]
-
- - NAME: random_world_rotation
- WORLD_ROT_ANGLE: [ -0.78539816, 0.78539816 ]
-
- - NAME: random_world_scaling
- WORLD_SCALE_RANGE: [ 0.95, 1.05 ]
-
-# anchor box related
-postprocess:
- core_method: 'VoxelPostprocessor' # VoxelPostprocessor, BevPostprocessor supported
- anchor_args:
- cav_lidar_range: *cav_lidar
- l: 3.9
- w: 1.6
- h: 1.56
- r: [0, 90]
- num: &achor_num 2
- target_args:
- pos_threshold: 0.6
- neg_threshold: 0.45
- score_threshold: 0.25
- order: 'hwl' # hwl or lwh
- max_num: 100 # maximum number of objects in a single frame. use this number to make sure different frames has the same dimension in the same batch
- nms_thresh: 0.15
-
-# model related
-model:
- core_method: voxel_net_intermediate # corresponding to VoxelNet
- args:
- N: *batch_size
- T: *T
- lidar_range: *cav_lidar
- voxel_size: *voxel_size
- anchor_num: *achor_num
- compression: 0
- pillar_vfe:
- use_norm: true
- with_distance: false
- use_absolute_xyz: true
- num_filters: [ 64 ]
-
-loss:
- core_method: point_pillar_loss
- args:
- cls_weight: 1.0
- reg: 2.0
-
-optimizer:
- core_method: Adam
- lr: 0.005
- args:
- eps: 1e-10
- weight_decay: 1e-4
-
-lr_scheduler:
- core_method: multistep #step, multistep and exponential and cosineannealwarm support
- gamma: 0.1
- step_size: [15, 30]
diff --git a/OpenCOOD/opencood/hypes_yaml/voxelnet_late_fusion.yaml b/OpenCOOD/opencood/hypes_yaml/voxelnet_late_fusion.yaml
deleted file mode 100644
index 02a88947..00000000
--- a/OpenCOOD/opencood/hypes_yaml/voxelnet_late_fusion.yaml
+++ /dev/null
@@ -1,88 +0,0 @@
-name: voxelnet_late_fusion
-
-yaml_parser: "load_voxel_params"
-root_dir: "opv2v_data_dumping/train"
-validate_dir: "opv2v_data_dumping/validate"
-
-train_params:
- batch_size: &batch_size 4
- epoches: 30
- eval_freq: 1
- save_freq: 1
-
-fusion:
- core_method: 'LateFusionDataset' # LateFusionDataset, EarlyFusionDataset, IntermediateFusionDataset supported
- args: []
-
-# preprocess-related
-preprocess:
- # options: BasePreprocessor, VoxelPreprocessor, BevPreprocessor
- core_method: 'SpVoxelPreprocessor'
- args:
- voxel_size: &voxel_size [0.4, 0.4, 0.4]
- max_points_per_voxel: &T 32
- max_voxel_train: 16000
- max_voxel_test: 40000
- # lidar range for each individual cav.
- cav_lidar_range: &cav_lidar [-70.4, -40, -3, 70.4, 40, 1]
-
-data_augment:
- - NAME: random_world_flip
- ALONG_AXIS_LIST: [ 'x' ]
-
- - NAME: random_world_rotation
- WORLD_ROT_ANGLE: [ -0.78539816, 0.78539816 ]
-
- - NAME: random_world_scaling
- WORLD_SCALE_RANGE: [ 0.95, 1.05 ]
-
-# anchor box related
-postprocess:
- core_method: 'VoxelPostprocessor' # VoxelPostprocessor, BevPostprocessor supported
- anchor_args:
- cav_lidar_range: *cav_lidar
- l: 3.9
- w: 1.6
- h: 1.56
- r: [0, 90]
- num: &achor_num 2
- target_args:
- pos_threshold: 0.6
- neg_threshold: 0.45
- score_threshold: 0.25
- order: 'hwl' # hwl or lwh
- max_num: 100 # maximum number of objects in a single frame. use this number to make sure different frames has the same dimension in the same batch
- nms_thresh: 0.05
-
-# model related
-model:
- core_method: voxel_net # corresponding to VoxelNet
- args:
- N: *batch_size
- T: *T
- lidar_range: *cav_lidar
- voxel_size: *voxel_size
- anchor_num: *achor_num
- pillar_vfe:
- use_norm: true
- with_distance: false
- use_absolute_xyz: true
- num_filters: [ 64 ]
-
-loss:
- core_method: point_pillar_loss
- args:
- cls_weight: 1.0
- reg: 2.0
-
-optimizer:
- core_method: Adam
- lr: 0.002
- args:
- eps: 1e-10
- weight_decay: 1e-4
-
-lr_scheduler:
- core_method: multistep #step, multistep and exponential and cosineannealwarm support
- gamma: 0.1
- step_size: [15, 30]
diff --git a/OpenCOOD/opencood/hypes_yaml/yaml_utils.py b/OpenCOOD/opencood/hypes_yaml/yaml_utils.py
deleted file mode 100644
index 4ac4612e..00000000
--- a/OpenCOOD/opencood/hypes_yaml/yaml_utils.py
+++ /dev/null
@@ -1,252 +0,0 @@
-import re
-import yaml
-import os
-import math
-
-import numpy as np
-
-
-def load_yaml(file, opt=None):
- """
- Load yaml file and return a dictionary.
-
- Parameters
- ----------
- file : string
- yaml file path.
-
- opt : argparser
- Argparser.
- Returns
- -------
- param : dict
- A dictionary that contains defined parameters.
- """
- if opt and opt.model_dir:
- file = os.path.join(opt.model_dir, "config.yaml")
-
- stream = open(file, "r")
- loader = yaml.Loader
- loader.add_implicit_resolver(
- "tag:yaml.org,2002:float",
- re.compile(
- """^(?:
- [-+]?(?:[0-9][0-9_]*)\\.[0-9_]*(?:[eE][-+]?[0-9]+)?
- |[-+]?(?:[0-9][0-9_]*)(?:[eE][-+]?[0-9]+)
- |\\.[0-9_]+(?:[eE][-+][0-9]+)?
- |[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+\\.[0-9_]*
- |[-+]?\\.(?:inf|Inf|INF)
- |\\.(?:nan|NaN|NAN))$""",
- re.X,
- ),
- list("-+0123456789."),
- )
- param = yaml.load(stream, Loader=loader)
- if "yaml_parser" in param:
- # TODO: Replace eval with secure function
- param = eval(param["yaml_parser"])(param)
-
- return param
-
-
-def load_voxel_params(param): # noqa: DC02
- """
- Based on the lidar range and resolution of voxel, calcuate the anchor box
- and target resolution.
-
- Parameters
- ----------
- param : dict
- Original loaded parameter dictionary.
-
- Returns
- -------
- param : dict
- Modified parameter dictionary with new attribute `anchor_args[W][H][L]`
- """
- anchor_args = param["postprocess"]["anchor_args"]
- cav_lidar_range = anchor_args["cav_lidar_range"]
- voxel_size = param["preprocess"]["args"]["voxel_size"]
-
- vw = voxel_size[0]
- vh = voxel_size[1]
- vd = voxel_size[2]
-
- anchor_args["vw"] = vw
- anchor_args["vh"] = vh
- anchor_args["vd"] = vd
-
- anchor_args["W"] = int((cav_lidar_range[3] - cav_lidar_range[0]) / vw)
- anchor_args["H"] = int((cav_lidar_range[4] - cav_lidar_range[1]) / vh)
- anchor_args["D"] = int((cav_lidar_range[5] - cav_lidar_range[2]) / vd)
-
- param["postprocess"].update({"anchor_args": anchor_args})
-
- # sometimes we just want to visualize the data without implementing model
- if "model" in param:
- param["model"]["args"]["W"] = anchor_args["W"]
- param["model"]["args"]["H"] = anchor_args["H"]
- param["model"]["args"]["D"] = anchor_args["D"]
-
- return param
-
-
-def load_point_pillar_params(param): # noqa: DC02
- """
- Based on the lidar range and resolution of voxel, calcuate the anchor box
- and target resolution.
-
- Parameters
- ----------
- param : dict
- Original loaded parameter dictionary.
-
- Returns
- -------
- param : dict
- Modified parameter dictionary with new attribute.
- """
- cav_lidar_range = param["preprocess"]["cav_lidar_range"]
- voxel_size = param["preprocess"]["args"]["voxel_size"]
-
- grid_size = (np.array(cav_lidar_range[3:6]) - np.array(cav_lidar_range[0:3])) / np.array(voxel_size)
- grid_size = np.round(grid_size).astype(np.int64)
- param["model"]["args"]["point_pillar_scatter"]["grid_size"] = grid_size
-
- anchor_args = param["postprocess"]["anchor_args"]
-
- vw = voxel_size[0]
- vh = voxel_size[1]
- vd = voxel_size[2]
-
- anchor_args["vw"] = vw
- anchor_args["vh"] = vh
- anchor_args["vd"] = vd
-
- anchor_args["W"] = math.ceil((cav_lidar_range[3] - cav_lidar_range[0]) / vw)
- anchor_args["H"] = math.ceil((cav_lidar_range[4] - cav_lidar_range[1]) / vh)
- anchor_args["D"] = math.ceil((cav_lidar_range[5] - cav_lidar_range[2]) / vd)
-
- param["postprocess"].update({"anchor_args": anchor_args})
-
- return param
-
-
-def load_second_params(param): # noqa: DC02
- """
- Based on the lidar range and resolution of voxel, calcuate the anchor box
- and target resolution.
-
- Parameters
- ----------
- param : dict
- Original loaded parameter dictionary.
-
- Returns
- -------
- param : dict
- Modified parameter dictionary with new attribute.
- """
- cav_lidar_range = param["preprocess"]["cav_lidar_range"]
- voxel_size = param["preprocess"]["args"]["voxel_size"]
-
- grid_size = (np.array(cav_lidar_range[3:6]) - np.array(cav_lidar_range[0:3])) / np.array(voxel_size)
- grid_size = np.round(grid_size).astype(np.int64)
- param["model"]["args"]["grid_size"] = grid_size
-
- anchor_args = param["postprocess"]["anchor_args"]
-
- vw = voxel_size[0]
- vh = voxel_size[1]
- vd = voxel_size[2]
-
- anchor_args["vw"] = vw
- anchor_args["vh"] = vh
- anchor_args["vd"] = vd
-
- anchor_args["W"] = int((cav_lidar_range[3] - cav_lidar_range[0]) / vw)
- anchor_args["H"] = int((cav_lidar_range[4] - cav_lidar_range[1]) / vh)
- anchor_args["D"] = int((cav_lidar_range[5] - cav_lidar_range[2]) / vd)
-
- param["postprocess"].update({"anchor_args": anchor_args})
-
- return param
-
-
-def load_bev_params(param): # noqa: DC02
- """
- Load bev related geometry parameters s.t. boundary, resolutions, input
- shape, target shape etc.
-
- Parameters
- ----------
- param : dict
- Original loaded parameter dictionary.
-
- Returns
- -------
- param : dict
- Modified parameter dictionary with new attribute `geometry_param`.
-
- """
- res = param["preprocess"]["args"]["res"]
- L1, W1, H1, L2, W2, H2 = param["preprocess"]["cav_lidar_range"]
- downsample_rate = param["preprocess"]["args"]["downsample_rate"]
-
- def f(low, high, r):
- return int((high - low) / r)
-
- input_shape = (int((f(L1, L2, res))), int((f(W1, W2, res))), int((f(H1, H2, res)) + 1))
- label_shape = (int(input_shape[0] / downsample_rate), int(input_shape[1] / downsample_rate), 7)
- geometry_param = {
- "L1": L1,
- "L2": L2,
- "W1": W1,
- "W2": W2,
- "H1": H1,
- "H2": H2,
- "downsample_rate": downsample_rate,
- "input_shape": input_shape,
- "label_shape": label_shape,
- "res": res,
- }
- param["preprocess"]["geometry_param"] = geometry_param
- param["postprocess"]["geometry_param"] = geometry_param
- param["model"]["args"]["geometry_param"] = geometry_param
- return param
-
-
-def save_yaml(data, save_name):
- """
- Save the dictionary into a yaml file.
-
- Parameters
- ----------
- data : dict
- The dictionary contains all data.
-
- save_name : string
- Full path of the output yaml file.
- """
-
- with open(save_name, "w") as outfile:
- yaml.dump(data, outfile, default_flow_style=False)
-
-
-def save_yaml_wo_overwriting(data, save_name):
- """
- Save the yaml file without overwriting the existing one.
-
- Parameters
- ----------
- data : dict
- The dictionary contains all data.
-
- save_name : string
- Full path of the output yaml file.
- """
- if os.path.exists(save_name):
- prev_data = load_yaml(save_name)
- data = {**data, **prev_data}
-
- save_yaml(data, save_name)
diff --git a/OpenCOOD/opencood/loss/__init__.py b/OpenCOOD/opencood/loss/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/OpenCOOD/opencood/loss/ciassd_loss.py b/OpenCOOD/opencood/loss/ciassd_loss.py
deleted file mode 100644
index 4a0b273e..00000000
--- a/OpenCOOD/opencood/loss/ciassd_loss.py
+++ /dev/null
@@ -1,223 +0,0 @@
-import torch
-import torch.nn as nn
-
-from opencood.data_utils.post_processor.voxel_postprocessor import VoxelPostprocessor
-from opencood.pcdet_utils.iou3d_nms.iou3d_nms_utils import aligned_boxes_iou3d_gpu
-
-
-class CiassdLoss(nn.Module):
- def __init__(self, args):
- super(CiassdLoss, self).__init__()
- self.pos_cls_weight = args["pos_cls_weight"]
- self.encode_rad_error_by_sin = args["encode_rad_error_by_sin"]
- self.cls = args["cls"]
- self.reg = args["reg"]
- self.iou = args["iou"]
- self.dir = args["dir"]
- self.loss_dict = {}
- ##
- self.num_cls = 2
- self.box_codesize = 7
-
- def forward(self, output_dict, label_dict):
- """
- Parameters
- ----------
- output_dict : dict
- target_dict : dict
- """
- preds_dict = output_dict["preds_dict_stage1"]
- target_dict = label_dict["stage1"]
- if "record_len" in output_dict:
- batch_size = int(output_dict["record_len"].sum())
- else:
- batch_size = output_dict["batch_size"]
-
- # ########
- # pred = torch.sigmoid(preds_dict['cls_preds'][0]).sum(dim=0).cpu().detach().numpy()
- # tagt_pos = target_dict['pos_equal_one'][0].sum(dim=-1).cpu().detach().numpy()
- # tagt_neg = target_dict['neg_equal_one'][0].sum(dim=-1).cpu().detach().numpy()
- # import matplotlib.pyplot as plt
- #
- # fig = plt.figure(figsize=(18, 8))
- # ax1 = fig.add_subplot(3, 1, 1)
- # ax1.imshow(pred, cmap='cool')
- # ax2 = fig.add_subplot(3, 1, 2)
- # ax2.imshow(tagt_pos, cmap='cool')
- # ax3 = fig.add_subplot(3, 1, 3)
- # ax3.imshow(tagt_neg, cmap='cool')
- # plt.show()
- # plt.close()
- # #########
-
- cls_labls = target_dict["pos_equal_one"].view(batch_size, -1, self.num_cls - 1)
- positives = cls_labls > 0
- negatives = target_dict["neg_equal_one"].view(batch_size, -1, self.num_cls - 1) > 0
- cared = torch.logical_or(positives, negatives)
- cls_labls = cls_labls * cared.type_as(cls_labls)
- # num_normalizer = cared.sum(1, keepdim=True)
- pos_normalizer = positives.sum(1, keepdim=True).float()
-
- # cls loss
- cls_preds = preds_dict["cls_preds"].permute(0, 2, 3, 1).contiguous().view(batch_size, -1, self.num_cls - 1)
- cls_weights = positives * self.pos_cls_weight + negatives * 1.0
- cls_weights /= torch.clamp(pos_normalizer, min=1.0)
- cls_loss = sigmoid_focal_loss(cls_preds, cls_labls, weights=cls_weights, **self.cls)
- cls_loss_reduced = cls_loss.sum() * self.cls["weight"] / batch_size
-
- # reg loss
- reg_weights = positives / torch.clamp(pos_normalizer, min=1.0)
- reg_preds = preds_dict["box_preds"].permute(0, 2, 3, 1).contiguous().view(batch_size, -1, self.box_codesize)
- reg_targets = target_dict["targets"].view(batch_size, -1, self.box_codesize)
- if self.encode_rad_error_by_sin:
- reg_preds, reg_targets = add_sin_difference(reg_preds, reg_targets)
- reg_loss = weighted_smooth_l1_loss(reg_preds, reg_targets, weights=reg_weights, sigma=self.reg["sigma"])
- reg_loss_reduced = reg_loss.sum() * self.reg["weight"] / batch_size
-
- # dir loss
- dir_targets = get_direction_target(reg_targets, output_dict["anchor_box"])
- dir_logits = preds_dict["dir_cls_preds"].permute(0, 2, 3, 1).contiguous().view(batch_size, -1, 2)
- dir_weights = (cls_labls > 0).type_as(dir_logits).view(batch_size, -1)
- dir_weights /= torch.clamp(dir_weights.sum(-1, keepdim=True), min=1.0) # [8, 70400], averaged in sample.
- dir_loss = softmax_cross_entropy_with_logits(dir_logits.view(-1, self.num_cls), dir_targets.view(-1, self.num_cls))
- dir_loss = dir_loss.view(dir_weights.shape) * dir_weights
- dir_loss_reduced = dir_loss.sum() * self.dir["weight"] / batch_size
-
- # iou loss
- iou_preds = preds_dict["iou_preds"].permute(0, 2, 3, 1).contiguous()
- pos_pred_mask = reg_weights.squeeze(dim=-1) > 0 # (4, 70400)
- iou_pos_preds = iou_preds.view(batch_size, -1)[pos_pred_mask]
- boxes3d_pred = VoxelPostprocessor.delta_to_boxes3d(
- preds_dict["box_preds"].permute(0, 2, 3, 1).contiguous().detach(), output_dict["anchor_box"], False
- )[pos_pred_mask]
- boxes3d_tgt = VoxelPostprocessor.delta_to_boxes3d(target_dict["targets"], output_dict["anchor_box"], False)[pos_pred_mask]
- iou_weights = reg_weights[pos_pred_mask].view(-1)
- iou_pos_targets = (
- aligned_boxes_iou3d_gpu(boxes3d_pred.float()[:, [0, 1, 2, 5, 4, 3, 6]], boxes3d_tgt.float()[:, [0, 1, 2, 5, 4, 3, 6]]).detach().squeeze()
- )
- iou_pos_targets = 2 * iou_pos_targets.view(-1) - 1
- iou_loss = weighted_smooth_l1_loss(iou_pos_preds, iou_pos_targets, weights=iou_weights, sigma=self.iou["sigma"])
-
- iou_loss_reduced = iou_loss.sum() * self.iou["weight"] / batch_size
-
- loss = cls_loss_reduced + reg_loss_reduced + dir_loss_reduced + iou_loss_reduced
-
- self.loss_dict.update(
- {
- "total_loss": loss,
- "cls_loss": cls_loss_reduced,
- "reg_loss": reg_loss_reduced,
- "dir_loss": dir_loss_reduced,
- "iou_loss": iou_loss_reduced,
- }
- )
-
- return loss
-
- def logging(self, epoch, batch_id, batch_len, writer, pbar=None):
- """
- Print out the loss function for current iteration.
-
- Parameters
- ----------
- epoch : int
- Current epoch for training.
- batch_id : int
- The current batch.
- batch_len : int
- Total batch length in one iteration of training,
- writer : SummaryWriter
- Used to visualize on tensorboard
- """
- total_loss = self.loss_dict["total_loss"]
- reg_loss = self.loss_dict["reg_loss"]
- cls_loss = self.loss_dict["cls_loss"]
- dir_loss = self.loss_dict["dir_loss"]
- iou_loss = self.loss_dict["iou_loss"]
- if (batch_id + 1) % 10 == 0:
- print(
- "[epoch %d][%d/%d], || Loss: %.4f || Cls: %.4f"
- " || Loc: %.4f || Dir: %.4f || Iou: %.4f"
- % (epoch, batch_id + 1, batch_len, total_loss.item(), cls_loss.item(), reg_loss.item(), dir_loss.item(), iou_loss.item())
- )
-
- writer.add_scalar("Regression_loss", reg_loss.item(), epoch * batch_len + batch_id)
- writer.add_scalar("Confidence_loss", cls_loss.item(), epoch * batch_len + batch_id)
- writer.add_scalar("Direction_loss", dir_loss.item(), epoch * batch_len + batch_id)
- writer.add_scalar("Iou_loss", iou_loss.item(), epoch * batch_len + batch_id)
-
-
-def add_sin_difference(boxes1, boxes2):
- rad_pred_encoding = torch.sin(boxes1[..., -1:]) * torch.cos(boxes2[..., -1:]) # ry -> sin(pred_ry)*cos(gt_ry)
- rad_gt_encoding = torch.cos(boxes1[..., -1:]) * torch.sin(boxes2[..., -1:]) # ry -> cos(pred_ry)*sin(gt_ry)
- res_boxes1 = torch.cat([boxes1[..., :-1], rad_pred_encoding], dim=-1)
- res_boxes2 = torch.cat([boxes2[..., :-1], rad_gt_encoding], dim=-1)
- return res_boxes1, res_boxes2
-
-
-def get_direction_target(reg_targets, anchors, one_hot=True, dir_offset=0.0):
- """
- Generate targets for bounding box direction classification.
-
- Parameters
- ----------
- anchors: torch.Tensor
- shape as (H*W*2, 7) or (H, W, 2, 7)
- reg_targets: torch.Tensor
- shape as (B, H*W*2, 7)
-
- Returns
- -------
- dir_cls_targets : torch.Tensor
- [batch_size, w*h*num_anchor_per_pos, 2]
- """
- batch_size = reg_targets.shape[0]
- anchors = anchors.view(1, -1, anchors.shape[-1]).repeat(batch_size, 1, 1)
- rot_gt = reg_targets[..., -1] + anchors[..., -1] # [4, 70400]
- dir_cls_targets = ((rot_gt - dir_offset) > 0).long() # [4, 70400]
- if one_hot:
- dir_cls_targets = one_hot_f(dir_cls_targets, 2, dtype=anchors.dtype)
- return dir_cls_targets
-
-
-def one_hot_f(tensor, depth, dim=-1, on_value=1.0, dtype=torch.float32):
- tensor_onehot = torch.zeros(*list(tensor.shape), depth, dtype=dtype, device=tensor.device) # [4, 70400, 2]
- tensor_onehot.scatter_(dim, tensor.unsqueeze(dim).long(), on_value) # [4, 70400, 2]
- return tensor_onehot
-
-
-def sigmoid_focal_loss(preds, targets, weights=None, **kwargs):
- assert "gamma" in kwargs and "alpha" in kwargs
- # sigmoid cross entropy with logits
- # more details: https://www.tensorflow.org/api_docs/python/tf/nn/sigmoid_cross_entropy_with_logits
- per_entry_cross_ent = torch.clamp(preds, min=0) - preds * targets.type_as(preds)
- per_entry_cross_ent += torch.log1p(torch.exp(-torch.abs(preds)))
- # focal loss
- prediction_probabilities = torch.sigmoid(preds)
- p_t = (targets * prediction_probabilities) + ((1 - targets) * (1 - prediction_probabilities))
- modulating_factor = torch.pow(1.0 - p_t, kwargs["gamma"])
- alpha_weight_factor = targets * kwargs["alpha"] + (1 - targets) * (1 - kwargs["alpha"])
-
- loss = modulating_factor * alpha_weight_factor * per_entry_cross_ent
- if weights is not None:
- loss *= weights
- return loss
-
-
-def softmax_cross_entropy_with_logits(logits, labels):
- param = list(range(len(logits.shape)))
- transpose_param = [0] + [param[-1]] + param[1:-1]
- logits = logits.permute(*transpose_param)
- loss_ftor = torch.nn.CrossEntropyLoss(reduction="none")
- loss = loss_ftor(logits, labels.max(dim=-1)[1])
- return loss
-
-
-def weighted_smooth_l1_loss(preds, targets, sigma=3.0, weights=None):
- diff = preds - targets
- abs_diff = torch.abs(diff)
- abs_diff_lt_1 = torch.le(abs_diff, 1 / (sigma**2)).type_as(abs_diff)
- loss = abs_diff_lt_1 * 0.5 * torch.pow(abs_diff * sigma, 2) + (abs_diff - 0.5 / (sigma**2)) * (1.0 - abs_diff_lt_1)
- if weights is not None:
- loss *= weights
- return loss
diff --git a/OpenCOOD/opencood/loss/fpvrcnn_loss.py b/OpenCOOD/opencood/loss/fpvrcnn_loss.py
deleted file mode 100644
index 5fc73928..00000000
--- a/OpenCOOD/opencood/loss/fpvrcnn_loss.py
+++ /dev/null
@@ -1,175 +0,0 @@
-import torch
-from torch import nn
-import numpy as np
-from opencood.loss.ciassd_loss import CiassdLoss, weighted_smooth_l1_loss
-
-
-class FpvrcnnLoss(nn.Module):
- def __init__(self, args):
- super(FpvrcnnLoss, self).__init__()
- self.ciassd_loss = CiassdLoss(args["stage1"])
- self.cls = args["stage2"]["cls"]
- self.reg = args["stage2"]["reg"]
- self.iou = args["stage2"]["iou"]
- self.loss_dict = {}
-
- def forward(self, output_dict, label_dict):
- """
- Parameters
- ----------
- output_dict : dict
- target_dict : dict
- """
- ciassd_loss = self.ciassd_loss(output_dict, label_dict)
-
- # only update ciassd if no bbox is detected in the first stage
- if "fpvrcnn_out" not in output_dict:
- self.loss_dict = {
- "loss": ciassd_loss,
- }
- return ciassd_loss
-
- # rcnn out
- rcnn_cls = output_dict["fpvrcnn_out"]["rcnn_cls"].view(1, -1, 1)
- rcnn_iou = output_dict["fpvrcnn_out"]["rcnn_iou"].view(1, -1, 1)
- rcnn_reg = output_dict["fpvrcnn_out"]["rcnn_reg"].view(1, -1, 7)
-
- tgt_cls = output_dict["rcnn_label_dict"]["cls_tgt"].view(1, -1, 1)
- tgt_iou = output_dict["rcnn_label_dict"]["iou_tgt"].view(1, -1, 1)
- tgt_reg = output_dict["rcnn_label_dict"]["reg_tgt"].view(1, -1, 7)
-
- _ = tgt_cls.sum() # pos_norm
- # cls loss
- loss_cls = weighted_sigmoid_binary_cross_entropy(rcnn_cls, tgt_cls)
-
- # iou loss
- # TODO: also count the negative samples
- loss_iou = weighted_smooth_l1_loss(rcnn_iou, tgt_iou, weights=tgt_cls).mean()
-
- # regression loss
- # Target resampling : Generate a weights mask to force the regressor concentrate on low iou predictions
- # sample 50% with iou>0.7 and 50% < 0.7
- weights = torch.ones(tgt_iou.shape, device=tgt_iou.device)
- weights[tgt_cls == 0] = 0
- neg = torch.logical_and(tgt_iou < 0.7, tgt_cls != 0)
- pos = torch.logical_and(tgt_iou >= 0.7, tgt_cls != 0)
- num_neg = int(neg.sum(dim=1))
- num_pos = int(pos.sum(dim=1))
- num_pos_smps = max(num_neg, 2)
- pos_indices = torch.where(pos)[1]
- not_selsected = torch.randperm(num_pos)[: num_pos - num_pos_smps]
- # not_selsected_indices = pos_indices[not_selsected]
- weights[:, pos_indices[not_selsected]] = 0
- loss_reg = weighted_smooth_l1_loss(rcnn_reg, tgt_reg, weights=weights / max(weights.sum(), 1)).sum()
-
- loss_cls_reduced = loss_cls * self.cls["weight"]
- loss_iou_reduced = loss_iou * self.iou["weight"]
- loss_reg_reduced = loss_reg * self.reg["weight"]
-
- # if torch.isnan(loss_reg_reduced):
- # print('debug')
-
- rcnn_loss = loss_cls_reduced + loss_iou_reduced + loss_reg_reduced
- loss = rcnn_loss + ciassd_loss
-
- self.loss_dict.update(
- {
- "loss": loss,
- "rcnn_loss": rcnn_loss,
- "cls_loss": loss_cls_reduced,
- "iou_loss": loss_iou_reduced,
- "reg_loss": loss_reg_reduced,
- }
- )
-
- return loss
-
- def logging(self, epoch, batch_id, batch_len, writer, pbar=None):
- """
- Print out the loss function for current iteration.
-
- Parameters
- ----------
- epoch : int
- Current epoch for training.
- batch_id : int
- The current batch.
- batch_len : int
- Total batch length in one iteration of training,
- writer : SummaryWriter
- Used to visualize on tensorboard
- """
- ciassd_loss_dict = self.ciassd_loss.loss_dict
- ciassd_total_loss = ciassd_loss_dict["total_loss"]
- reg_loss = ciassd_loss_dict["reg_loss"]
- cls_loss = ciassd_loss_dict["cls_loss"]
- dir_loss = ciassd_loss_dict["dir_loss"]
- iou_loss = ciassd_loss_dict["iou_loss"]
-
- if (batch_id + 1) % 10 == 0:
- str_to_print = "[epoch %d][%d/%d], || Loss: %.4f || Ciassd: %.4f || Cls1: %.4f || Loc1: %.4f || Dir1: %.4f || Iou1: %.4f" % (
- epoch,
- batch_id + 1,
- batch_len,
- self.loss_dict["loss"],
- ciassd_total_loss.item(),
- cls_loss.item(),
- reg_loss.item(),
- dir_loss.item(),
- iou_loss.item(),
- )
- if "rcnn_loss" in self.loss_dict:
- str_to_print += " || Rcnn: %.4f || Cls2: %.4f || Loc2: %.4f || Iou2: %.4f" % (
- self.loss_dict["rcnn_loss"],
- self.loss_dict["cls_loss"].item(),
- self.loss_dict["reg_loss"].item(),
- self.loss_dict["iou_loss"].item(),
- )
- print(str_to_print)
-
- writer.add_scalar("Ciassd_regression_loss", reg_loss.item(), epoch * batch_len + batch_id)
- writer.add_scalar("Ciassd_Confidence_loss", cls_loss.item(), epoch * batch_len + batch_id)
- writer.add_scalar("Ciassd_Direction_loss", dir_loss.item(), epoch * batch_len + batch_id)
- writer.add_scalar("Ciassd_Iou_loss", iou_loss.item(), epoch * batch_len + batch_id)
- writer.add_scalar("Ciassd_loss", ciassd_total_loss.item(), epoch * batch_len + batch_id)
- if "rcnn_loss" in self.loss_dict:
- writer.add_scalar("Rcnn_regression_loss", self.loss_dict["reg_loss"].item(), epoch * batch_len + batch_id)
- writer.add_scalar("Rcnn_Confidence_loss", self.loss_dict["cls_loss"].item(), epoch * batch_len + batch_id)
- writer.add_scalar("Rcnn_Iou_loss", self.loss_dict["iou_loss"].item(), epoch * batch_len + batch_id)
- writer.add_scalar("Rcnn_loss", self.loss_dict["rcnn_loss"].item(), epoch * batch_len + batch_id)
- writer.add_scalar("Total_loss", self.loss_dict["loss"].item(), epoch * batch_len + batch_id)
-
-
-def weighted_sigmoid_binary_cross_entropy(preds, tgts, weights=None, class_indices=None):
- if weights is not None:
- weights = weights.unsqueeze(-1)
- if class_indices is not None:
- weights *= indices_to_dense_vector(class_indices, preds.shape[2]).view(1, 1, -1).type_as(preds)
- per_entry_cross_ent = nn.functional.binary_cross_entropy_with_logits(preds, tgts, weights)
- return per_entry_cross_ent
-
-
-def indices_to_dense_vector(indices, size, indices_value=1.0, default_value=0, dtype=np.float32):
- """Creates dense vector with indices set to specific value and rest to zeros.
-
- This function exists because it is unclear if it is safe to use
- tf.sparse_to_dense(indices, [size], 1, validate_indices=False)
- with indices which are not ordered.
- This function accepts a dynamic size (e.g. tf.shape(tensor)[0])
-
- Args:
- indices: 1d Tensor with integer indices which are to be set to
- indices_values.
- size: scalar with size (integer) of output Tensor.
- indices_value: values of elements specified by indices in the output vector
- default_value: values of other elements in the output vector.
- dtype: data type.
-
- Returns:
- dense 1D Tensor of shape [size] with indices set to indices_values and the
- rest set to default_value.
- """
- dense = torch.zeros(size).fill_(default_value)
- dense[indices] = indices_value
-
- return dense
diff --git a/OpenCOOD/opencood/loss/pixor_loss.py b/OpenCOOD/opencood/loss/pixor_loss.py
deleted file mode 100644
index da8ff4d2..00000000
--- a/OpenCOOD/opencood/loss/pixor_loss.py
+++ /dev/null
@@ -1,116 +0,0 @@
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-
-
-class PixorLoss(nn.Module):
- def __init__(self, args):
- super(PixorLoss, self).__init__()
- self.alpha = args["alpha"]
- self.beta = args["beta"]
- self.loss_dict = {}
-
- def dtype(self):
- return torch.float16
-
- def forward(self, output_dict, target_dict):
- """
- Compute loss for pixor network
- Parameters
- ----------
- output_dict : dict
- The dictionary that contains the output.
-
- target_dict : dict
- The dictionary that contains the target.
-
- Returns
- -------
- total_loss : torch.Tensor
- Total loss.
-
- """
- targets = target_dict["label_map"]
- cls_preds, loc_preds = output_dict["cls"], output_dict["reg"]
-
- cls_targets, loc_targets = targets.split([1, 6], dim=1)
- pos_count = cls_targets.sum()
- neg_count = (cls_targets == 0).sum()
-
- w1 = (neg_count / (pos_count + neg_count)).to(dtype=self.dtype())
- w2 = (pos_count / (pos_count + neg_count)).to(dtype=self.dtype())
-
- weights = torch.ones_like(cls_preds.reshape(-1), dtype=self.dtype(), device=cls_preds.device)
- weights[cls_targets.reshape(-1) == 1] = w1
- weights[cls_targets.reshape(-1) == 0] = w2
-
- # cls_loss = F.binary_cross_entropy_with_logits(input=cls_preds.reshape(-1), target=cls_targets.reshape(-1), weight=weights,
- # reduction='mean')
-
- print(f"cls_preds shape: {cls_preds.shape}")
- print(f"cls_targets shape: {cls_targets.shape}")
- # print(f"pos_neg_weights shape: {pos_neg_weights.shape}")
-
- if cls_targets.shape[2:] != cls_preds.shape[2:]:
- cls_targets = F.interpolate(cls_targets, size=cls_preds.shape[2:], mode="bilinear", align_corners=False)
- # pos_neg_weights = F.interpolate(pos_neg_weights, size=cls_preds.shape[2:], mode='bilinear', align_corners=False)
- print(f"Resized cls_targets shape: {cls_targets.shape}")
- # print(f"Resized pos_neg_weights shape: {pos_neg_weights.shape}")
-
- cls_loss = F.binary_cross_entropy_with_logits(input=cls_preds, target=cls_targets, reduction="mean")
- pos_pixels = cls_targets.sum()
-
- loc_loss = F.smooth_l1_loss(cls_targets * loc_preds, cls_targets * loc_targets, reduction="sum")
- loc_loss = loc_loss / pos_pixels if pos_pixels > 0 else loc_loss
-
- total_loss = self.alpha * cls_loss + self.beta * loc_loss
-
- self.loss_dict.update({"total_loss": total_loss, "reg_loss": loc_loss, "cls_loss": cls_loss})
-
- return total_loss
-
- def logging(self, epoch, batch_id, batch_len, writer, pbar=None):
- """
- Print out the loss function for current iteration.
-
- Parameters
- ----------
- epoch : int
- Current epoch for training.
- batch_id : int
- The current batch.
- batch_len : int
- Total batch length in one iteration of training,
- writer : SummaryWriter
- Used to visualize on tensorboard
- """
- total_loss = self.loss_dict["total_loss"]
- reg_loss = self.loss_dict["reg_loss"]
- cls_loss = self.loss_dict["cls_loss"]
-
- if pbar is None:
- print(
- "[epoch %d][%d/%d], || Loss: %.4f || Conf Loss: %.4f"
- " || Loc Loss: %.4f" % (epoch, batch_id + 1, batch_len, total_loss.item(), cls_loss.item(), reg_loss.item())
- )
- else:
- pbar.set_description(
- "[epoch %d][%d/%d], || Loss: %.4f || Conf Loss: %.4f"
- " || Loc Loss: %.4f" % (epoch, batch_id + 1, batch_len, total_loss.item(), cls_loss.item(), reg_loss.item())
- )
-
- writer.add_scalar("Regression_loss", reg_loss.item(), epoch * batch_len + batch_id)
- writer.add_scalar("Confidence_loss", cls_loss.item(), epoch * batch_len + batch_id)
-
-
-def test():
- torch.manual_seed(0)
- loss = PixorLoss(None)
- pred = torch.sigmoid(torch.randn(1, 7, 2, 3))
- label = torch.zeros(1, 7, 2, 3)
- loss = loss(pred, label)
- print(loss)
-
-
-if __name__ == "__main__":
- test()
diff --git a/OpenCOOD/opencood/loss/point_pillar_loss.py b/OpenCOOD/opencood/loss/point_pillar_loss.py
deleted file mode 100644
index 6b976123..00000000
--- a/OpenCOOD/opencood/loss/point_pillar_loss.py
+++ /dev/null
@@ -1,222 +0,0 @@
-import torch
-import torch.nn as nn
-import numpy as np
-
-
-class WeightedSmoothL1Loss(nn.Module):
- """
- Code-wise Weighted Smooth L1 Loss modified based on fvcore.nn.smooth_l1_loss
- https://github.com/facebookresearch/fvcore/blob/master/fvcore/nn/smooth_l1_loss.py
- | 0.5 * x ** 2 / beta if abs(x) < beta
- smoothl1(x) = |
- | abs(x) - 0.5 * beta otherwise,
- where x = input - target.
- """
-
- def __init__(self, beta: float = 1.0 / 9.0, code_weights: list = None):
- """
- Args:
- beta: Scalar float.
- L1 to L2 change point.
- For beta values < 1e-5, L1 loss is computed.
- code_weights: (#codes) float list if not None.
- Code-wise weights.
- """
- super(WeightedSmoothL1Loss, self).__init__()
- self.beta = beta
- if code_weights is not None:
- self.code_weights = np.array(code_weights, dtype=np.float32)
- self.code_weights = torch.from_numpy(self.code_weights).cuda()
-
- @staticmethod
- def smooth_l1_loss(diff, beta):
- if beta < 1e-5:
- loss = torch.abs(diff)
- else:
- n = torch.abs(diff)
- loss = torch.where(n < beta, 0.5 * n**2 / beta, n - 0.5 * beta)
-
- return loss
-
- def forward(self, input: torch.Tensor, target: torch.Tensor, weights: torch.Tensor = None):
- """
- Args:
- input: (B, #anchors, #codes) float tensor.
- Ecoded predicted locations of objects.
- target: (B, #anchors, #codes) float tensor.
- Regression targets.
- weights: (B, #anchors) float tensor if not None.
-
- Returns:
- loss: (B, #anchors) float tensor.
- Weighted smooth l1 loss without reduction.
- """
- target = torch.where(torch.isnan(target), input, target) # ignore nan targets
-
- diff = input - target
- loss = self.smooth_l1_loss(diff, self.beta)
-
- # anchor-wise weighting
- if weights is not None:
- assert weights.shape[0] == loss.shape[0] and weights.shape[1] == loss.shape[1]
- loss = loss * weights.unsqueeze(-1)
-
- return loss
-
-
-class PointPillarLoss(nn.Module):
- def __init__(self, args):
- super(PointPillarLoss, self).__init__()
- self.reg_loss_func = WeightedSmoothL1Loss()
- self.alpha = 0.25
- self.gamma = 2.0
-
- self.cls_weight = args["cls_weight"]
- self.reg_coe = args["reg"]
- self.loss_dict = {}
-
- def forward(self, output_dict, target_dict):
- """
- Parameters
- ----------
- output_dict : dict
- target_dict : dict
- """
- rm = output_dict["rm"]
- psm = output_dict["psm"]
- targets = target_dict["targets"]
-
- cls_preds = psm.permute(0, 2, 3, 1).contiguous()
-
- box_cls_labels = target_dict["pos_equal_one"]
- box_cls_labels = box_cls_labels.view(psm.shape[0], -1).contiguous()
-
- positives = box_cls_labels > 0
- negatives = box_cls_labels == 0
- negative_cls_weights = negatives * 1.0
- cls_weights = (negative_cls_weights + 1.0 * positives).float()
- reg_weights = positives.float()
-
- pos_normalizer = positives.sum(1, keepdim=True).float()
- reg_weights /= torch.clamp(pos_normalizer, min=1.0)
- cls_weights /= torch.clamp(pos_normalizer, min=1.0)
- cls_targets = box_cls_labels
- cls_targets = cls_targets.unsqueeze(dim=-1)
-
- cls_targets = cls_targets.squeeze(dim=-1)
- one_hot_targets = torch.zeros(*list(cls_targets.shape), 2, dtype=cls_preds.dtype, device=cls_targets.device)
- one_hot_targets.scatter_(-1, cls_targets.unsqueeze(dim=-1).long(), 1.0)
- cls_preds = cls_preds.view(psm.shape[0], -1, 1)
- one_hot_targets = one_hot_targets[..., 1:]
-
- cls_loss_src = self.cls_loss_func(cls_preds, one_hot_targets, weights=cls_weights) # [N, M]
- cls_loss = cls_loss_src.sum() / psm.shape[0]
- conf_loss = cls_loss * self.cls_weight
-
- # regression
- rm = rm.permute(0, 2, 3, 1).contiguous()
- rm = rm.view(rm.size(0), -1, 7)
- targets = targets.view(targets.size(0), -1, 7)
- box_preds_sin, reg_targets_sin = self.add_sin_difference(rm, targets)
- loc_loss_src = self.reg_loss_func(box_preds_sin, reg_targets_sin, weights=reg_weights)
- reg_loss = loc_loss_src.sum() / rm.shape[0]
- reg_loss *= self.reg_coe
-
- total_loss = reg_loss + conf_loss
-
- self.loss_dict.update({"total_loss": total_loss, "reg_loss": reg_loss, "conf_loss": conf_loss})
-
- return total_loss
-
- def cls_loss_func(self, input: torch.Tensor, target: torch.Tensor, weights: torch.Tensor):
- """
- Args:
- input: (B, #anchors, #classes) float tensor.
- Predicted logits for each class
- target: (B, #anchors, #classes) float tensor.
- One-hot encoded classification targets
- weights: (B, #anchors) float tensor.
- Anchor-wise weights.
-
- Returns:
- weighted_loss: (B, #anchors, #classes) float tensor after weighting.
- """
- pred_sigmoid = torch.sigmoid(input)
- alpha_weight = target * self.alpha + (1 - target) * (1 - self.alpha)
- pt = target * (1.0 - pred_sigmoid) + (1.0 - target) * pred_sigmoid
- focal_weight = alpha_weight * torch.pow(pt, self.gamma)
-
- bce_loss = self.sigmoid_cross_entropy_with_logits(input, target)
-
- loss = focal_weight * bce_loss
-
- if weights.shape.__len__() == 2 or (weights.shape.__len__() == 1 and target.shape.__len__() == 2):
- weights = weights.unsqueeze(-1)
-
- assert weights.shape.__len__() == loss.shape.__len__()
-
- return loss * weights
-
- @staticmethod
- def sigmoid_cross_entropy_with_logits(input: torch.Tensor, target: torch.Tensor):
- """PyTorch Implementation for tf.nn.sigmoid_cross_entropy_with_logits:
- max(x, 0) - x * z + log(1 + exp(-abs(x))) in
- https://www.tensorflow.org/api_docs/python/tf/nn/sigmoid_cross_entropy_with_logits
-
- Args:
- input: (B, #anchors, #classes) float tensor.
- Predicted logits for each class
- target: (B, #anchors, #classes) float tensor.
- One-hot encoded classification targets
-
- Returns:
- loss: (B, #anchors, #classes) float tensor.
- Sigmoid cross entropy loss without reduction
- """
- loss = torch.clamp(input, min=0) - input * target + torch.log1p(torch.exp(-torch.abs(input)))
- return loss
-
- @staticmethod
- def add_sin_difference(boxes1, boxes2, dim=6):
- assert dim != -1
- rad_pred_encoding = torch.sin(boxes1[..., dim : dim + 1]) * torch.cos(boxes2[..., dim : dim + 1])
- rad_tg_encoding = torch.cos(boxes1[..., dim : dim + 1]) * torch.sin(boxes2[..., dim : dim + 1])
-
- boxes1 = torch.cat([boxes1[..., :dim], rad_pred_encoding, boxes1[..., dim + 1 :]], dim=-1)
- boxes2 = torch.cat([boxes2[..., :dim], rad_tg_encoding, boxes2[..., dim + 1 :]], dim=-1)
- return boxes1, boxes2
-
- def logging(self, epoch, batch_id, batch_len, writer, pbar=None):
- """
- Print out the loss function for current iteration.
-
- Parameters
- ----------
- epoch : int
- Current epoch for training.
- batch_id : int
- The current batch.
- batch_len : int
- Total batch length in one iteration of training,
- writer : SummaryWriter
- Used to visualize on tensorboard
- pbar : tqdm
- Progress bar
- """
- total_loss = self.loss_dict["total_loss"]
- reg_loss = self.loss_dict["reg_loss"]
- conf_loss = self.loss_dict["conf_loss"]
-
- if pbar is None:
- print(
- "[epoch %d][%d/%d], || Loss: %.4f || Conf Loss: %.4f"
- " || Loc Loss: %.4f" % (epoch, batch_id + 1, batch_len, total_loss.item(), conf_loss.item(), reg_loss.item())
- )
- else:
- pbar.set_description(
- "[epoch %d][%d/%d], || Loss: %.4f || Conf Loss: %.4f"
- " || Loc Loss: %.4f" % (epoch, batch_id + 1, batch_len, total_loss.item(), conf_loss.item(), reg_loss.item())
- )
-
- writer.add_scalar("Regression_loss", reg_loss.item(), epoch * batch_len + batch_id)
- writer.add_scalar("Confidence_loss", conf_loss.item(), epoch * batch_len + batch_id)
diff --git a/OpenCOOD/opencood/models/__init__.py b/OpenCOOD/opencood/models/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/OpenCOOD/opencood/models/ciassd.py b/OpenCOOD/opencood/models/ciassd.py
deleted file mode 100644
index b9f0bdbf..00000000
--- a/OpenCOOD/opencood/models/ciassd.py
+++ /dev/null
@@ -1,44 +0,0 @@
-from torch import nn
-import numpy as np
-
-from opencood.models.sub_modules.mean_vfe import MeanVFE
-from opencood.models.sub_modules.sparse_backbone_3d import VoxelBackBone8x
-from opencood.models.sub_modules.height_compression import HeightCompression
-from opencood.models.sub_modules.cia_ssd_utils import SSFA, Head
-
-
-class CIASSD(nn.Module):
- def __init__(self, args):
- super(CIASSD, self).__init__()
- lidar_range = np.array(args["lidar_range"])
- grid_size = np.round((lidar_range[3:6] - lidar_range[:3]) / np.array(args["voxel_size"])).astype(np.int64)
- self.vfe = MeanVFE(args["mean_vfe"], args["mean_vfe"]["num_point_features"])
- self.spconv_block = VoxelBackBone8x(args["spconv"], input_channels=args["spconv"]["num_features_in"], grid_size=grid_size)
- self.map_to_bev = HeightCompression(args["map2bev"])
- self.ssfa = SSFA(args["ssfa"])
- self.head = Head(**args["head"])
-
- def forward(self, batch_dict):
- processed_lidar = batch_dict["processed_lidar"]
- batch_size = batch_dict.get("batch_size")
- if batch_size is None:
- batch_size = batch_dict["object_bbx_center"].shape[0]
- model_batch = {
- **processed_lidar,
- "batch_size": batch_size,
- }
- model_batch = self.vfe(model_batch)
- model_batch = self.spconv_block(model_batch)
- model_batch = self.map_to_bev(model_batch)
- out = self.ssfa(model_batch["spatial_features"])
- out = self.head(out)
- return {
- "preds_dict_stage1": out,
- "batch_size": batch_size,
- "anchor_box": batch_dict.get("anchor_box"),
- }
-
-
-if __name__ == "__main__":
- model = SSFA(None)
- print(model)
diff --git a/OpenCOOD/opencood/models/communication_adapters/__init__.py b/OpenCOOD/opencood/models/communication_adapters/__init__.py
deleted file mode 100644
index 2db04053..00000000
--- a/OpenCOOD/opencood/models/communication_adapters/__init__.py
+++ /dev/null
@@ -1,121 +0,0 @@
-"""Model-owned communication adapter selection."""
-
-from typing import cast
-
-import torch
-from torch import nn
-
-from opencood.models.communication_adapters.base import ModelCommunicationAdapter
-from opencood.models.communication_adapters.early import EarlyFusionCommunicationAdapter
-from opencood.models.communication_adapters.inference_input import (
- BevInferenceInput,
- DetectionInferenceInput,
- FpvrcnnAgentInferenceInput,
- InferenceInput,
- MultiScaleFeatureInput,
- PointCloudInferenceInput,
- SpatialFeatureInput,
- VoxelInferenceInput,
- Where2CommFeatureInput,
- build_inference_input,
- inference_input_to_dict,
- merge_inference_inputs,
-)
-from opencood.models.communication_adapters.intermediate import (
- IntermediateFeatureCommunicationAdapter,
- MultiScaleFeatureCommunicationAdapter,
- SpatialFeatureCommunicationAdapter,
-)
-from opencood.models.communication_adapters.inference_metadata import (
- IntermediateMetadata,
- PoseFrameMetadata,
- V2XViTMetadata,
-)
-from opencood.models.communication_adapters.late import LateFusionCommunicationAdapter
-from opencood.models.communication_adapters.where2comm import Where2CommCommunicationAdapter
-from opencood.models.communication_adapters.wire_payload import (
- EarlyFusionWirePayload,
- FpvrcnnWirePayload,
- IntermediateFeatureWirePayload,
- LateFusionWirePayload,
- Where2CommWirePayload,
-)
-
-
-def build_communication_adapter(
- model: nn.Module,
- device: torch.device,
- fusion_method: str | None = None,
-) -> ModelCommunicationAdapter:
- """
- Build the communication adapter declared by a perception model.
-
- Fusion-wide contracts are selected from the configured fusion method.
- Intermediate models declare the adapter matching their learned-feature
- communication boundary.
-
- Parameters
- ----------
- model : torch.nn.Module
- Cooperative perception model instance.
- device : torch.device
- Device used for sender-side model execution.
- fusion_method : str | None
- Explicit fusion method selected by the model configuration.
-
- Returns
- -------
- ModelCommunicationAdapter
- Adapter responsible for the model's network communication boundary.
-
- Raises
- ------
- TypeError
- If the model declares an invalid adapter class.
- """
- if fusion_method == "EarlyFusionDataset":
- return EarlyFusionCommunicationAdapter(model, device)
- if fusion_method == "LateFusionDataset":
- return LateFusionCommunicationAdapter(model, device)
-
- adapter_class: object = getattr(type(model), "communication_adapter_class", None)
- if adapter_class is None:
- raise TypeError(f"{type(model).__name__} must declare communication_adapter_class")
- if not isinstance(adapter_class, type) or not issubclass(
- adapter_class,
- ModelCommunicationAdapter,
- ):
- raise TypeError("model.communication_adapter_class must derive from ModelCommunicationAdapter")
- return cast(type[ModelCommunicationAdapter], adapter_class)(model, device)
-
-
-__all__ = (
- "BevInferenceInput",
- "DetectionInferenceInput",
- "EarlyFusionWirePayload",
- "EarlyFusionCommunicationAdapter",
- "FpvrcnnAgentInferenceInput",
- "FpvrcnnWirePayload",
- "InferenceInput",
- "IntermediateFeatureCommunicationAdapter",
- "IntermediateFeatureWirePayload",
- "IntermediateMetadata",
- "LateFusionWirePayload",
- "LateFusionCommunicationAdapter",
- "ModelCommunicationAdapter",
- "MultiScaleFeatureCommunicationAdapter",
- "MultiScaleFeatureInput",
- "PointCloudInferenceInput",
- "PoseFrameMetadata",
- "SpatialFeatureCommunicationAdapter",
- "SpatialFeatureInput",
- "V2XViTMetadata",
- "VoxelInferenceInput",
- "Where2CommCommunicationAdapter",
- "Where2CommFeatureInput",
- "Where2CommWirePayload",
- "build_communication_adapter",
- "build_inference_input",
- "inference_input_to_dict",
- "merge_inference_inputs",
-)
diff --git a/OpenCOOD/opencood/models/communication_adapters/base.py b/OpenCOOD/opencood/models/communication_adapters/base.py
deleted file mode 100644
index 137de7fa..00000000
--- a/OpenCOOD/opencood/models/communication_adapters/base.py
+++ /dev/null
@@ -1,107 +0,0 @@
-"""Common interface for model-specific network communication adapters."""
-
-from abc import ABC, abstractmethod
-from typing import Any
-
-import torch
-from torch import nn
-
-
-class ModelCommunicationAdapter(ABC):
- """Bridge a perception model to its sender/receiver wire contract."""
-
- def __init__(self, model: nn.Module, device: torch.device) -> None:
- self.model = model
- self.device = device
-
- @abstractmethod
- def prepare_transmission_payloads(self, dataset: Any, idx: int) -> None:
- """
- Build and publish every sender payload for one dataset frame.
-
- Parameters
- ----------
- dataset : Any
- OpenCOOD dataset providing the local scene observations.
- idx : int
- Dataset frame to encode and publish.
- """
-
- def move_to_device(self, value: Any) -> Any:
- """
- Recursively move a model input to the adapter device.
-
- Parameters
- ----------
- value : Any
- Nested tensors, mappings, or sequences forming a model input.
-
- Returns
- -------
- Any
- Input with every tensor moved to the configured device.
- """
- if isinstance(value, torch.Tensor):
- return value.to(self.device)
- if isinstance(value, dict):
- return {key: self.move_to_device(item) for key, item in value.items()}
- if isinstance(value, list):
- return [self.move_to_device(item) for item in value]
- if isinstance(value, tuple):
- return tuple(self.move_to_device(item) for item in value)
- return value
-
- def decode_received_payload(
- self,
- payload: object,
- receiver_pose: list[float],
- ) -> dict[str, Any]:
- """
- Decode one delivered model payload for receiver-side inference.
-
- Parameters
- ----------
- payload : object
- Deserialized model payload delivered over the V2X communication
- channel.
- receiver_pose : list[float]
- Receiver pose used for model-specific coordinate conversion.
-
- Returns
- -------
- dict[str, Any]
- Model-specific remote inference input.
-
- Raises
- ------
- NotImplementedError
- If the adapter does not support receiver-side payload decoding.
- """
- raise NotImplementedError(f"{type(self).__name__} does not implement receiver-side payload decoding")
-
- def encode_local_intermediate_input(
- self,
- dataset: Any,
- inference_input: Any,
- ) -> dict[str, Any]:
- """
- Encode the receiver's local input at an intermediate model boundary.
-
- Parameters
- ----------
- dataset : Any
- Dataset providing model preprocessing facilities.
- inference_input : Any
- Receiver-local preprocessor output.
-
- Returns
- -------
- dict[str, Any]
- Model-specific learned feature input.
-
- Raises
- ------
- NotImplementedError
- If the adapter does not implement an intermediate feature split.
- """
- raise NotImplementedError(f"{type(self).__name__} does not implement intermediate feature encoding")
diff --git a/OpenCOOD/opencood/models/communication_adapters/early.py b/OpenCOOD/opencood/models/communication_adapters/early.py
deleted file mode 100644
index 53c28606..00000000
--- a/OpenCOOD/opencood/models/communication_adapters/early.py
+++ /dev/null
@@ -1,111 +0,0 @@
-"""Early-fusion V2X communication adapter."""
-
-from collections.abc import Callable
-from typing import Any, Protocol, runtime_checkable
-
-import numpy as np
-import numpy.typing as npt
-
-from opencood.models.communication_adapters.base import ModelCommunicationAdapter
-from opencood.models.communication_adapters.inference_metadata import PoseFrameMetadata
-from opencood.models.communication_adapters.wire_payload import EarlyFusionWirePayload
-from opencood.utils import box_utils
-from opencood.utils.transformation_utils import x1_to_x2
-
-
-@runtime_checkable
-class EarlyFusionCommunicationDataset(Protocol):
- """Dataset capabilities required by the early-fusion adapter."""
-
- def extract_data(
- self,
- idx: int,
- agent_payload_builder: Callable[[npt.NDArray[Any], PoseFrameMetadata], object],
- ) -> None:
- raise NotImplementedError
-
-
-class EarlyFusionCommunicationAdapter(ModelCommunicationAdapter):
- """Exchange local LiDAR points and project them only at the receiver."""
-
- def prepare_transmission_payloads(self, dataset: Any, idx: int) -> None:
- """
- Build and publish an early-fusion payload for every local agent.
-
- Parameters
- ----------
- dataset : Any
- Dataset exposing local early-fusion agent observations.
- idx : int
- Dataset frame to publish.
-
- Raises
- ------
- TypeError
- If the selected dataset cannot build early-fusion observations.
- """
- if not isinstance(dataset, EarlyFusionCommunicationDataset):
- raise TypeError("Early fusion requires a dataset capable of building local LiDAR observations")
- dataset.extract_data(idx, agent_payload_builder=self._build_wire_payload)
-
- @staticmethod
- def _build_wire_payload(
- lidar_points: npt.NDArray[Any],
- metadata: PoseFrameMetadata,
- ) -> EarlyFusionWirePayload:
- """
- Build a payload without applying a receiver-specific transformation.
-
- Parameters
- ----------
- lidar_points : numpy.typing.NDArray[Any]
- LiDAR points expressed in the sender coordinate frame.
- metadata : PoseFrameMetadata
- Sender pose and capture frame.
-
- Returns
- -------
- EarlyFusionWirePayload
- Local LiDAR points and the metadata needed by a receiver.
- """
- return EarlyFusionWirePayload(
- lidar_points=lidar_points,
- metadata=metadata,
- )
-
- def decode_received_payload(
- self,
- payload: object,
- receiver_pose: list[float],
- ) -> dict[str, Any]:
- """
- Project one sender-local point cloud into the receiver frame.
-
- Parameters
- ----------
- payload : object
- Deserialized early-fusion payload delivered over V2X.
- receiver_pose : list[float]
- Receiver LiDAR pose for the current inference frame.
-
- Returns
- -------
- dict[str, Any]
- Point cloud expressed in the receiver coordinate frame.
-
- Raises
- ------
- TypeError
- If a payload belonging to another contract is received.
- """
- if not isinstance(payload, EarlyFusionWirePayload):
- raise TypeError(f"Expected EarlyFusionWirePayload, got {type(payload).__name__}")
-
- transformation_matrix = x1_to_x2(payload.metadata.lidar_pose, receiver_pose)
- projected_lidar = np.array(payload.lidar_points, copy=True)
- if len(projected_lidar) > 0:
- projected_lidar[:, :3] = box_utils.project_points_by_matrix_torch(
- projected_lidar[:, :3],
- transformation_matrix,
- )
- return {"projected_lidar": projected_lidar}
diff --git a/OpenCOOD/opencood/models/communication_adapters/fpvrcnn.py b/OpenCOOD/opencood/models/communication_adapters/fpvrcnn.py
deleted file mode 100644
index d45578ae..00000000
--- a/OpenCOOD/opencood/models/communication_adapters/fpvrcnn.py
+++ /dev/null
@@ -1,202 +0,0 @@
-"""Communication adapter for distributed FPV-RCNN inference."""
-
-from collections.abc import Callable, Mapping
-from typing import Any, Protocol, runtime_checkable
-
-import numpy as np
-import numpy.typing as npt
-import torch
-
-from opencood.models.communication_adapters.base import ModelCommunicationAdapter
-from opencood.models.communication_adapters.inference_input import (
- FpvrcnnAgentInferenceInput,
-)
-from opencood.models.communication_adapters.inference_metadata import PoseFrameMetadata
-from opencood.models.communication_adapters.wire_payload import FpvrcnnWirePayload
-from opencood.utils import box_utils
-from opencood.utils.transformation_utils import x1_to_x2
-
-
-@runtime_checkable
-class FpvrcnnCommunicationDataset(Protocol):
- """Dataset capabilities required by the FPV-RCNN adapter."""
-
- pre_processor: Any
- post_processor: Any
-
- def extract_data(
- self,
- idx: int,
- agent_payload_builder: Callable[[dict[str, Any], PoseFrameMetadata], object] | None,
- ) -> None:
- raise NotImplementedError
-
-
-class FpvrcnnCommunicationAdapter(ModelCommunicationAdapter):
- """Run FPV-RCNN stage one and VSA before serializing an agent payload."""
-
- def prepare_transmission_payloads(self, dataset: Any, idx: int) -> None:
- """
- Encode and publish FPV-RCNN outputs for every local agent.
-
- Parameters
- ----------
- dataset : Any
- Dataset exposing the FPV-RCNN agent-sample capabilities.
- idx : int
- Dataset frame to encode and publish.
-
- Raises
- ------
- TypeError
- If the selected dataset cannot prepare FPV-RCNN agent samples.
- """
- if not isinstance(dataset, FpvrcnnCommunicationDataset):
- raise TypeError("FPV-RCNN communication requires a dataset capable of building local agent observations")
- dataset.extract_data(
- idx,
- agent_payload_builder=lambda sample, metadata: self._build_wire_payload(
- dataset,
- sample,
- metadata,
- ),
- )
-
- def _build_wire_payload(
- self,
- dataset: FpvrcnnCommunicationDataset,
- sample: dict[str, Any],
- metadata: PoseFrameMetadata,
- ) -> FpvrcnnWirePayload:
- """
- Run sender-side FPV-RCNN and build its typed wire payload.
-
- Parameters
- ----------
- dataset : FpvrcnnCommunicationDataset
- Dataset providing the FPV-RCNN preprocessor and anchors.
- sample : dict[str, Any]
- Local sender model input without supervision fields.
- metadata : PoseFrameMetadata
- Sender pose and feature capture frame.
-
- Returns
- -------
- FpvrcnnWirePayload
- Typed payload containing proposals and VSA keypoint features.
-
- Raises
- ------
- TypeError
- If the model output is not the expected tensor mapping.
- """
- encode_agent = getattr(self.model, "encode_agent", None)
- if encode_agent is None:
- raise TypeError("FpvrcnnCommunicationAdapter requires a model with encode_agent()")
-
- processed_lidar = dataset.pre_processor.collate_batch([sample["processed_features"]])
- origin_lidar = np.pad(sample["projected_lidar"], ((0, 0), (1, 0)), mode="constant")
- batch = self.move_to_device(
- {
- "processed_lidar": processed_lidar,
- "origin_lidar": torch.from_numpy(origin_lidar),
- "anchor_box": torch.from_numpy(np.asarray(dataset.post_processor.generate_anchor_box())),
- "record_len": torch.ones(1, dtype=torch.long),
- }
- )
- self.model.eval()
- with torch.no_grad():
- output = encode_agent(batch)
- if not isinstance(output, Mapping):
- raise TypeError("FPV-RCNN encode_agent() must return a tensor mapping")
-
- return FpvrcnnWirePayload(
- inference_input=FpvrcnnAgentInferenceInput(
- proposals=self._tensor_to_numpy(output, "proposals"),
- proposal_scores=self._tensor_to_numpy(output, "proposal_scores"),
- point_coords=self._tensor_to_numpy(output, "point_coords"),
- point_features=self._tensor_to_numpy(output, "point_features"),
- ),
- metadata=metadata,
- )
-
- def decode_received_payload(
- self,
- payload: object,
- receiver_pose: list[float],
- ) -> dict[str, Any]:
- """
- Decode and project one FPV-RCNN payload into the receiver frame.
-
- Parameters
- ----------
- payload : object
- Deserialized FPV-RCNN payload delivered over V2X.
- receiver_pose : list[float]
- Receiver LiDAR pose for the current inference frame.
-
- Returns
- -------
- dict[str, Any]
- Remote proposals and VSA keypoint data in the receiver frame.
-
- Raises
- ------
- TypeError
- If a payload belonging to another model contract is received.
- """
- if not isinstance(payload, FpvrcnnWirePayload):
- raise TypeError(f"Expected FpvrcnnWirePayload, got {type(payload).__name__}")
-
- transformation_matrix = x1_to_x2(payload.metadata.lidar_pose, receiver_pose)
- point_coords = np.array(payload.inference_input.point_coords, copy=True)
- if len(point_coords) > 0:
- point_coords[:, :3] = box_utils.project_points_by_matrix_torch(
- point_coords[:, :3],
- transformation_matrix,
- )
-
- proposals = np.array(payload.inference_input.proposals, copy=True)
- if len(proposals) > 0:
- proposals[:, :3] = box_utils.project_points_by_matrix_torch(
- proposals[:, :3],
- transformation_matrix,
- )
- proposals[:, 6] += np.arctan2(
- transformation_matrix[1, 0],
- transformation_matrix[0, 0],
- )
-
- return {
- "proposals": proposals,
- "proposal_scores": np.array(payload.inference_input.proposal_scores, copy=True),
- "point_coords": point_coords,
- "point_features": np.array(payload.inference_input.point_features, copy=True),
- }
-
- @staticmethod
- def _tensor_to_numpy(output: Mapping[str, Any], field: str) -> npt.NDArray[Any]:
- """
- Detach one required model output as a NumPy array.
-
- Parameters
- ----------
- output : collections.abc.Mapping[str, Any]
- Sender-side model output.
- field : str
- Required tensor field to convert.
-
- Returns
- -------
- numpy.typing.NDArray[Any]
- Detached CPU representation of the tensor.
-
- Raises
- ------
- TypeError
- If the required field is not a tensor.
- """
- value = output.get(field)
- if not isinstance(value, torch.Tensor):
- raise TypeError(f'FPV-RCNN output field "{field}" must be a tensor')
- return value.detach().cpu().numpy()
diff --git a/OpenCOOD/opencood/models/communication_adapters/inference_input.py b/OpenCOOD/opencood/models/communication_adapters/inference_input.py
deleted file mode 100644
index 22e3962b..00000000
--- a/OpenCOOD/opencood/models/communication_adapters/inference_input.py
+++ /dev/null
@@ -1,253 +0,0 @@
-"""Typed model inputs exchanged through communication adapters."""
-
-from dataclasses import dataclass
-from typing import Any, Mapping, TypeAlias
-
-import numpy.typing as npt
-
-
-VOXEL_INPUT_FIELDS = frozenset({"voxel_features", "voxel_coords"})
-SPARSE_VOXEL_INPUT_FIELDS = VOXEL_INPUT_FIELDS | {"voxel_num_points"}
-BEV_INPUT_FIELDS = frozenset({"bev_input"})
-POINT_CLOUD_INPUT_FIELDS = frozenset({"downsample_lidar"})
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class VoxelInferenceInput:
- """
- Voxelized LiDAR input.
-
- Attributes
- ----------
- voxel_features : numpy.typing.NDArray[Any]
- Features stored for every non-empty voxel.
- voxel_coords : numpy.typing.NDArray[Any]
- Coordinates of the non-empty voxels.
- voxel_num_points : numpy.typing.NDArray[Any] | None
- Number of points in every voxel when supplied by the preprocessor.
- """
-
- voxel_features: npt.NDArray[Any]
- voxel_coords: npt.NDArray[Any]
- voxel_num_points: npt.NDArray[Any] | None = None
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class BevInferenceInput:
- """
- Bird's-eye-view model input.
-
- Attributes
- ----------
- bev_input : numpy.typing.NDArray[Any]
- Dense BEV feature map produced by the preprocessor.
- """
-
- bev_input: npt.NDArray[Any]
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class PointCloudInferenceInput:
- """
- Point-cloud model input.
-
- Attributes
- ----------
- points : numpy.typing.NDArray[Any]
- Sampled LiDAR points produced by the preprocessor.
- """
-
- points: npt.NDArray[Any]
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class FpvrcnnAgentInferenceInput:
- """
- Agent-side FPV-RCNN output consumed by cooperative stage two.
-
- Attributes
- ----------
- proposals : numpy.typing.NDArray[Any]
- Stage-one bounding-box proposals in the sender coordinate frame.
- proposal_scores : numpy.typing.NDArray[Any]
- Confidence scores corresponding to ``proposals``.
- point_coords : numpy.typing.NDArray[Any]
- Coordinates of VSA keypoints in the sender coordinate frame.
- point_features : numpy.typing.NDArray[Any]
- VSA features corresponding to ``point_coords``.
- """
-
- proposals: npt.NDArray[Any]
- proposal_scores: npt.NDArray[Any]
- point_coords: npt.NDArray[Any]
- point_features: npt.NDArray[Any]
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class DetectionInferenceInput:
- """
- Detection candidates produced by one late-fusion sender.
-
- Attributes
- ----------
- boxes : numpy.typing.NDArray[Any]
- Bounding-box corners expressed in the sender coordinate frame.
- scores : numpy.typing.NDArray[Any]
- Confidence scores corresponding to ``boxes``.
- """
-
- boxes: npt.NDArray[Any]
- scores: npt.NDArray[Any]
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class Where2CommFeatureInput:
- """
- Learned feature data selected by a Where2Comm sender.
-
- Attributes
- ----------
- spatial_features : numpy.typing.NDArray[Any]
- Sender-side feature representation after the communication mask and
- optional learned compression encoder.
- communication_rate : float
- Fraction of spatial locations selected for transmission.
- """
-
- spatial_features: npt.NDArray[Any]
- communication_rate: float
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class SpatialFeatureInput:
- """
- Single learned spatial feature map produced by an agent encoder.
-
- Attributes
- ----------
- spatial_features : numpy.typing.NDArray[Any]
- Feature map including its single-agent batch dimension.
- """
-
- spatial_features: npt.NDArray[Any]
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class MultiScaleFeatureInput:
- """
- Learned feature maps produced at multiple encoder scales.
-
- Attributes
- ----------
- feature_maps : tuple[numpy.typing.NDArray[Any], ...]
- Ordered feature maps including their single-agent batch dimensions.
- """
-
- feature_maps: tuple[npt.NDArray[Any], ...]
-
-
-InferenceInput: TypeAlias = VoxelInferenceInput | BevInferenceInput | PointCloudInferenceInput
-
-
-def build_inference_input(preprocessor_output: Mapping[str, Any]) -> InferenceInput:
- """
- Convert a preprocessor result into a typed inference input.
-
- The concrete type is selected from the fields actually returned by the
- preprocessor. Dataset and transport code therefore remain independent of
- the configured preprocessor class.
-
- Parameters
- ----------
- preprocessor_output : collections.abc.Mapping[str, Any]
- Result returned by ``preprocessor.preprocess()``.
-
- Returns
- -------
- InferenceInput
- Typed representation of the preprocessor result.
-
- Raises
- ------
- ValueError
- If the returned field set does not match a supported input type.
- """
- fields = frozenset(preprocessor_output)
-
- if fields == VOXEL_INPUT_FIELDS or fields == SPARSE_VOXEL_INPUT_FIELDS:
- return VoxelInferenceInput(
- voxel_features=preprocessor_output["voxel_features"],
- voxel_coords=preprocessor_output["voxel_coords"],
- voxel_num_points=preprocessor_output.get("voxel_num_points"),
- )
- if fields == BEV_INPUT_FIELDS:
- return BevInferenceInput(bev_input=preprocessor_output["bev_input"])
- if fields == POINT_CLOUD_INPUT_FIELDS:
- return PointCloudInferenceInput(points=preprocessor_output["downsample_lidar"])
-
- raise ValueError(f"Unsupported preprocessor output fields: {sorted(fields)}")
-
-
-def inference_input_to_dict(inference_input: InferenceInput) -> dict[str, npt.NDArray[Any]]:
- """
- Convert a typed inference input to the preprocessor collate format.
-
- Parameters
- ----------
- inference_input : InferenceInput
- Typed model input.
-
- Returns
- -------
- dict[str, numpy.typing.NDArray[Any]]
- Mapping accepted by the corresponding preprocessor's collate path.
- """
- if isinstance(inference_input, VoxelInferenceInput):
- output = {
- "voxel_features": inference_input.voxel_features,
- "voxel_coords": inference_input.voxel_coords,
- }
- if inference_input.voxel_num_points is not None:
- output["voxel_num_points"] = inference_input.voxel_num_points
- return output
- if isinstance(inference_input, BevInferenceInput):
- return {"bev_input": inference_input.bev_input}
- if isinstance(inference_input, PointCloudInferenceInput):
- return {"downsample_lidar": inference_input.points}
- raise TypeError(f"Unsupported inference input type: {type(inference_input).__name__}")
-
-
-def merge_inference_inputs(inference_inputs: list[InferenceInput]) -> dict[str, list[npt.NDArray[Any]]]:
- """
- Merge homogeneous agent inputs for preprocessor collation.
-
- Parameters
- ----------
- inference_inputs : list[InferenceInput]
- Inputs for the agents participating in the current inference.
-
- Returns
- -------
- dict[str, list[numpy.typing.NDArray[Any]]]
- Dict-of-lists accepted by ``preprocessor.collate_batch()``.
-
- Raises
- ------
- ValueError
- If the input list is empty.
- TypeError
- If agents provide different input representations.
- """
- if not inference_inputs:
- raise ValueError("At least one inference input is required")
-
- input_type = type(inference_inputs[0])
- if any(type(inference_input) is not input_type for inference_input in inference_inputs[1:]):
- raise TypeError("All agents must use the same inference input representation")
-
- input_dicts = [inference_input_to_dict(inference_input) for inference_input in inference_inputs]
- input_fields = set(input_dicts[0])
- if any(set(input_dict) != input_fields for input_dict in input_dicts[1:]):
- raise TypeError("All agents must provide the same inference input fields")
-
- return {field: [input_dict[field] for input_dict in input_dicts] for field in input_dicts[0]}
diff --git a/OpenCOOD/opencood/models/communication_adapters/inference_metadata.py b/OpenCOOD/opencood/models/communication_adapters/inference_metadata.py
deleted file mode 100644
index b351b37e..00000000
--- a/OpenCOOD/opencood/models/communication_adapters/inference_metadata.py
+++ /dev/null
@@ -1,49 +0,0 @@
-"""Typed metadata exchanged through communication adapters."""
-
-from dataclasses import dataclass
-from typing import Any, TypeAlias
-
-import numpy.typing as npt
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class PoseFrameMetadata:
- """
- Pose and capture frame used to derive pairwise transformations.
-
- Attributes
- ----------
- lidar_pose : list[float] | tuple[float, ...]
- Agent LiDAR pose at feature capture time.
- capture_frame : int
- Dataset frame in which the model input was captured.
- """
-
- lidar_pose: list[float] | tuple[float, ...]
- capture_frame: int
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class V2XViTMetadata:
- """
- Metadata consumed by the V2X-ViT fusion model.
-
- Attributes
- ----------
- velocity : float
- Sender velocity normalized by 30 km/h.
- agent_type : int
- ``1`` for an infrastructure agent and ``0`` for a vehicle.
- capture_frame : int
- Dataset frame in which the model input was captured.
- spatial_correction_matrix : numpy.typing.NDArray[Any]
- Transformation correcting the captured feature to the receive frame.
- """
-
- velocity: float
- agent_type: int
- capture_frame: int
- spatial_correction_matrix: npt.NDArray[Any]
-
-
-IntermediateMetadata: TypeAlias = PoseFrameMetadata | V2XViTMetadata
diff --git a/OpenCOOD/opencood/models/communication_adapters/intermediate.py b/OpenCOOD/opencood/models/communication_adapters/intermediate.py
deleted file mode 100644
index e2a6adbb..00000000
--- a/OpenCOOD/opencood/models/communication_adapters/intermediate.py
+++ /dev/null
@@ -1,261 +0,0 @@
-"""Reusable adapters for learned intermediate feature maps."""
-
-from collections.abc import Callable, Mapping
-from typing import Any, Protocol, runtime_checkable
-
-import numpy as np
-import torch
-
-from opencood.models.communication_adapters.base import ModelCommunicationAdapter
-from opencood.models.communication_adapters.inference_input import (
- InferenceInput,
- MultiScaleFeatureInput,
- SpatialFeatureInput,
- inference_input_to_dict,
-)
-from opencood.models.communication_adapters.inference_metadata import IntermediateMetadata
-from opencood.models.communication_adapters.wire_payload import IntermediateFeatureWirePayload
-
-
-@runtime_checkable
-class IntermediateCommunicationDataset(Protocol):
- """Dataset capabilities required by learned-feature adapters."""
-
- pre_processor: Any
-
- def extract_data(
- self,
- idx: int,
- agent_payload_builder: Callable[
- [InferenceInput, IntermediateMetadata | None],
- object,
- ],
- ) -> None:
- raise NotImplementedError
-
-
-class IntermediateFeatureCommunicationAdapter(ModelCommunicationAdapter):
- """Base adapter for models exposing an ``encode_agent`` split."""
-
- def prepare_transmission_payloads(self, dataset: Any, idx: int) -> None:
- """
- Encode and publish learned features for every local agent.
-
- Parameters
- ----------
- dataset : Any
- Intermediate dataset exposing individual agent inputs.
- idx : int
- Dataset frame to publish.
- """
- communication_dataset = self._require_dataset(dataset)
- communication_dataset.extract_data(
- idx,
- agent_payload_builder=lambda inference_input, metadata: self._build_wire_payload(
- communication_dataset,
- inference_input,
- metadata,
- ),
- )
-
- @staticmethod
- def _require_dataset(dataset: Any) -> IntermediateCommunicationDataset:
- if not isinstance(dataset, IntermediateCommunicationDataset):
- raise TypeError("Intermediate feature encoding requires an intermediate communication dataset")
- return dataset
-
- def _encode(
- self,
- dataset: IntermediateCommunicationDataset,
- inference_input: InferenceInput,
- ) -> Mapping[str, Any]:
- encode_agent = getattr(self.model, "encode_agent", None)
- if encode_agent is None:
- raise TypeError(f"{type(self).__name__} requires a model with encode_agent()")
- model_input = self.move_to_device(
- {
- "processed_lidar": dataset.pre_processor.collate_batch([inference_input_to_dict(inference_input)]),
- "batch_size": 1,
- }
- )
- self.model.eval()
- with torch.no_grad():
- output = encode_agent(model_input)
- if not isinstance(output, Mapping):
- raise TypeError("encode_agent() must return a tensor mapping")
- return output
-
- def _build_wire_payload(
- self,
- dataset: IntermediateCommunicationDataset,
- inference_input: InferenceInput,
- metadata: IntermediateMetadata | None,
- ) -> IntermediateFeatureWirePayload:
- encoded = self._encode(dataset, inference_input)
- return IntermediateFeatureWirePayload(
- inference_input=self._serialize_features(encoded),
- metadata=metadata,
- )
-
- def encode_local_intermediate_input(
- self,
- dataset: Any,
- inference_input: Any,
- ) -> dict[str, Any]:
- """
- Encode the receiver's own observation at the model boundary.
-
- Parameters
- ----------
- dataset : Any
- Intermediate dataset providing preprocessing.
- inference_input : Any
- Receiver-local typed preprocessor output.
-
- Returns
- -------
- dict[str, Any]
- Learned feature maps in the receiver assembly format.
- """
- communication_dataset = self._require_dataset(dataset)
- encoded = self._encode(communication_dataset, inference_input)
- return self._deserialize_features(self._serialize_features(encoded))
-
- def decode_received_payload(
- self,
- payload: object,
- receiver_pose: list[float],
- ) -> dict[str, Any]:
- """
- Decode one successfully delivered intermediate feature payload.
-
- Parameters
- ----------
- payload : object
- Deserialized model payload delivered over V2X.
- receiver_pose : list[float]
- Receiver pose reserved for model-side alignment.
-
- Returns
- -------
- dict[str, Any]
- Learned features and their sender metadata.
- """
- del receiver_pose
- if not isinstance(payload, IntermediateFeatureWirePayload):
- raise TypeError(f"Expected IntermediateFeatureWirePayload, got {type(payload).__name__}")
- decoded = self._deserialize_features(payload.inference_input)
- decoded["metadata"] = payload.metadata
- return decoded
-
- def _serialize_features(
- self,
- encoded: Mapping[str, Any],
- ) -> SpatialFeatureInput | MultiScaleFeatureInput:
- raise NotImplementedError
-
- def _deserialize_features(
- self,
- inference_input: SpatialFeatureInput | MultiScaleFeatureInput,
- ) -> dict[str, Any]:
- raise NotImplementedError
-
-
-class SpatialFeatureCommunicationAdapter(IntermediateFeatureCommunicationAdapter):
- """Exchange one dense learned feature map per agent."""
-
- def _serialize_features(
- self,
- encoded: Mapping[str, Any],
- ) -> SpatialFeatureInput:
- """
- Move one encoded spatial feature map to the wire representation.
-
- Parameters
- ----------
- encoded : collections.abc.Mapping[str, Any]
- Model output containing ``spatial_features``.
-
- Returns
- -------
- SpatialFeatureInput
- CPU NumPy representation of the learned feature map.
- """
- spatial_features = encoded.get("spatial_features")
- if not isinstance(spatial_features, torch.Tensor):
- raise TypeError('encode_agent() field "spatial_features" must be a tensor')
- return SpatialFeatureInput(
- spatial_features=spatial_features.detach().cpu().numpy(),
- )
-
- def _deserialize_features(
- self,
- inference_input: SpatialFeatureInput | MultiScaleFeatureInput,
- ) -> dict[str, Any]:
- """
- Restore one spatial feature map for receiver-side collation.
-
- Parameters
- ----------
- inference_input : SpatialFeatureInput | MultiScaleFeatureInput
- Typed feature representation received over V2X.
-
- Returns
- -------
- dict[str, Any]
- Receiver assembly mapping containing ``spatial_features``.
- """
- if not isinstance(inference_input, SpatialFeatureInput):
- raise TypeError(f"Expected SpatialFeatureInput, got {type(inference_input).__name__}")
- return {
- "spatial_features": np.array(inference_input.spatial_features, copy=True),
- }
-
-
-class MultiScaleFeatureCommunicationAdapter(IntermediateFeatureCommunicationAdapter):
- """Exchange an ordered tuple of learned feature scales per agent."""
-
- def _serialize_features(
- self,
- encoded: Mapping[str, Any],
- ) -> MultiScaleFeatureInput:
- """
- Move ordered model feature scales to the wire representation.
-
- Parameters
- ----------
- encoded : collections.abc.Mapping[str, Any]
- Model output containing ``feature_maps``.
-
- Returns
- -------
- MultiScaleFeatureInput
- CPU NumPy representations of all learned feature scales.
- """
- feature_maps = encoded.get("feature_maps")
- if not isinstance(feature_maps, (list, tuple)) or not all(isinstance(feature_map, torch.Tensor) for feature_map in feature_maps):
- raise TypeError('encode_agent() field "feature_maps" must be a tensor sequence')
- return MultiScaleFeatureInput(
- feature_maps=tuple(feature_map.detach().cpu().numpy() for feature_map in feature_maps),
- )
-
- def _deserialize_features(
- self,
- inference_input: SpatialFeatureInput | MultiScaleFeatureInput,
- ) -> dict[str, Any]:
- """
- Restore ordered feature scales for receiver-side collation.
-
- Parameters
- ----------
- inference_input : SpatialFeatureInput | MultiScaleFeatureInput
- Typed feature representation received over V2X.
-
- Returns
- -------
- dict[str, Any]
- Receiver assembly mapping keyed by scale index.
- """
- if not isinstance(inference_input, MultiScaleFeatureInput):
- raise TypeError(f"Expected MultiScaleFeatureInput, got {type(inference_input).__name__}")
- return {f"feature_{index}": np.array(feature_map, copy=True) for index, feature_map in enumerate(inference_input.feature_maps)}
diff --git a/OpenCOOD/opencood/models/communication_adapters/late.py b/OpenCOOD/opencood/models/communication_adapters/late.py
deleted file mode 100644
index e065b762..00000000
--- a/OpenCOOD/opencood/models/communication_adapters/late.py
+++ /dev/null
@@ -1,170 +0,0 @@
-"""Late-fusion V2X communication adapter."""
-
-from collections.abc import Callable, Mapping
-from typing import Any, Protocol, runtime_checkable
-
-import numpy as np
-import torch
-
-from opencood.models.communication_adapters.base import ModelCommunicationAdapter
-from opencood.models.communication_adapters.inference_input import (
- DetectionInferenceInput,
- InferenceInput,
- inference_input_to_dict,
-)
-from opencood.models.communication_adapters.inference_metadata import PoseFrameMetadata
-from opencood.models.communication_adapters.wire_payload import LateFusionWirePayload
-from opencood.utils import box_utils
-from opencood.utils.transformation_utils import x1_to_x2
-
-
-@runtime_checkable
-class LateFusionCommunicationDataset(Protocol):
- """Dataset capabilities required by the late-fusion adapter."""
-
- pre_processor: Any
- post_processor: Any
-
- def extract_data(
- self,
- idx: int,
- agent_payload_builder: Callable[[InferenceInput, PoseFrameMetadata], object],
- ) -> None:
- raise NotImplementedError
-
-
-class LateFusionCommunicationAdapter(ModelCommunicationAdapter):
- """Run a complete detector locally and exchange detection candidates."""
-
- def prepare_transmission_payloads(self, dataset: Any, idx: int) -> None:
- """
- Run sender-side late-fusion inference for every local agent.
-
- Parameters
- ----------
- dataset : Any
- Dataset exposing local detector inputs and post-processing.
- idx : int
- Dataset frame to publish.
-
- Raises
- ------
- TypeError
- If the selected dataset lacks late-fusion capabilities.
- """
- if not isinstance(dataset, LateFusionCommunicationDataset):
- raise TypeError("Late fusion requires a dataset capable of building local detector inputs")
- dataset.extract_data(
- idx,
- agent_payload_builder=lambda inference_input, metadata: self._build_wire_payload(
- dataset,
- inference_input,
- metadata,
- ),
- )
-
- def _build_wire_payload(
- self,
- dataset: LateFusionCommunicationDataset,
- inference_input: InferenceInput,
- metadata: PoseFrameMetadata,
- ) -> LateFusionWirePayload:
- """
- Run the local detector and serialize its decoded candidates.
-
- Parameters
- ----------
- dataset : LateFusionCommunicationDataset
- Dataset providing pre- and post-processing implementations.
- inference_input : InferenceInput
- Preprocessed input belonging only to the sender.
- metadata : PoseFrameMetadata
- Sender pose and capture frame.
-
- Returns
- -------
- LateFusionWirePayload
- Decoded sender-frame boxes and their confidence scores.
-
- Raises
- ------
- TypeError
- If the model output or postprocessor result is invalid.
- """
- processed_lidar = dataset.pre_processor.collate_batch([inference_input_to_dict(inference_input)])
- anchor_box = dataset.post_processor.generate_anchor_box()
- model_input = self.move_to_device(
- {
- "processed_lidar": processed_lidar,
- "batch_size": 1,
- "anchor_box": (None if anchor_box is None else torch.from_numpy(np.asarray(anchor_box))),
- }
- )
-
- self.model.eval()
- with torch.no_grad():
- output = self.model(model_input)
- if not isinstance(output, Mapping):
- raise TypeError("Late-fusion model must return a tensor mapping")
-
- decode_predictions = getattr(dataset.post_processor, "decode_agent_predictions", None)
- if decode_predictions is None:
- raise TypeError("Late-fusion postprocessor must implement decode_agent_predictions()")
- boxes, scores = decode_predictions(model_input, output)
- if not isinstance(boxes, torch.Tensor) or not isinstance(scores, torch.Tensor):
- raise TypeError("decode_agent_predictions() must return box and score tensors")
-
- return LateFusionWirePayload(
- detections=DetectionInferenceInput(
- boxes=boxes.detach().cpu().numpy(),
- scores=scores.detach().cpu().numpy(),
- ),
- metadata=metadata,
- )
-
- def decode_received_payload(
- self,
- payload: object,
- receiver_pose: list[float],
- ) -> dict[str, Any]:
- """
- Project one sender's detections into the receiver frame.
-
- Parameters
- ----------
- payload : object
- Deserialized late-fusion payload delivered over V2X.
- receiver_pose : list[float]
- Receiver LiDAR pose for the current inference frame.
-
- Returns
- -------
- dict[str, Any]
- Detection candidates expressed in the receiver frame.
-
- Raises
- ------
- TypeError
- If the payload or box representation is unsupported.
- """
- if not isinstance(payload, LateFusionWirePayload):
- raise TypeError(f"Expected LateFusionWirePayload, got {type(payload).__name__}")
-
- boxes = np.array(payload.detections.boxes, copy=True)
- transformation_matrix = x1_to_x2(payload.metadata.lidar_pose, receiver_pose)
- if boxes.ndim != 3 or boxes.shape[-2:] not in {(4, 2), (8, 3)}:
- raise TypeError(f"Unsupported late-fusion box shape: {boxes.shape}")
- if boxes.shape[0] > 0:
- if boxes.shape[-1] == 3:
- boxes = box_utils.project_box3d(boxes, transformation_matrix)
- else:
- box3d = np.pad(boxes, ((0, 0), (0, 0), (0, 1)))
- boxes = box_utils.project_points_by_matrix_torch(
- box3d.reshape(-1, 3),
- transformation_matrix,
- )[:, :2].reshape(-1, 4, 2)
-
- return {
- "pred_box_tensor": boxes,
- "pred_score": np.array(payload.detections.scores, copy=True),
- }
diff --git a/OpenCOOD/opencood/models/communication_adapters/where2comm.py b/OpenCOOD/opencood/models/communication_adapters/where2comm.py
deleted file mode 100644
index 23c849ad..00000000
--- a/OpenCOOD/opencood/models/communication_adapters/where2comm.py
+++ /dev/null
@@ -1,226 +0,0 @@
-"""Where2Comm intermediate-feature communication adapter."""
-
-from collections.abc import Callable, Mapping
-from typing import Any, Protocol, runtime_checkable
-
-import numpy as np
-import torch
-
-from opencood.models.communication_adapters.base import ModelCommunicationAdapter
-from opencood.models.communication_adapters.inference_input import (
- InferenceInput,
- Where2CommFeatureInput,
- inference_input_to_dict,
-)
-from opencood.models.communication_adapters.inference_metadata import PoseFrameMetadata
-from opencood.models.communication_adapters.wire_payload import Where2CommWirePayload
-
-
-@runtime_checkable
-class Where2CommCommunicationDataset(Protocol):
- """Dataset capabilities required by the Where2Comm adapter."""
-
- pre_processor: Any
-
- def extract_data(
- self,
- idx: int,
- agent_payload_builder: Callable[
- [InferenceInput, PoseFrameMetadata],
- object,
- ],
- ) -> None:
- raise NotImplementedError
-
-
-class Where2CommCommunicationAdapter(ModelCommunicationAdapter):
- """Execute the private Where2Comm encoder before V2X transmission."""
-
- def prepare_transmission_payloads(self, dataset: Any, idx: int) -> None:
- """
- Encode and publish learned features for every local agent.
-
- Parameters
- ----------
- dataset : Any
- Dataset exposing individual intermediate-fusion inputs.
- idx : int
- Dataset frame to publish.
-
- Raises
- ------
- TypeError
- If the selected dataset lacks the required capabilities.
- """
- if not isinstance(dataset, Where2CommCommunicationDataset):
- raise TypeError("Where2Comm requires an intermediate-fusion communication dataset")
- dataset.extract_data(
- idx,
- agent_payload_builder=lambda inference_input, metadata: self._build_wire_payload(
- dataset,
- inference_input,
- metadata,
- ),
- )
-
- def _encode(
- self,
- dataset: Where2CommCommunicationDataset,
- inference_input: InferenceInput,
- *,
- apply_communication_mask: bool,
- ) -> tuple[np.ndarray, float]:
- """
- Run the sender-private model stages for one agent.
-
- Parameters
- ----------
- dataset : Where2CommCommunicationDataset
- Dataset providing the configured preprocessor.
- inference_input : InferenceInput
- One agent's preprocessed LiDAR input.
- apply_communication_mask : bool
- Whether to apply the over-the-air feature selection mask.
-
- Returns
- -------
- tuple[numpy.ndarray, float]
- Learned feature map and selected-location ratio.
-
- Raises
- ------
- TypeError
- If the model does not expose the Where2Comm encoder contract.
- """
- encode_agent = getattr(self.model, "encode_agent", None)
- if encode_agent is None:
- raise TypeError("Where2CommCommunicationAdapter requires a model with encode_agent()")
-
- model_input = self.move_to_device(
- {
- "processed_lidar": dataset.pre_processor.collate_batch([inference_input_to_dict(inference_input)]),
- }
- )
- self.model.eval()
- with torch.no_grad():
- output = encode_agent(
- model_input,
- apply_communication_mask=apply_communication_mask,
- )
- if not isinstance(output, Mapping):
- raise TypeError("Where2Comm encode_agent() must return a tensor mapping")
- spatial_features = output.get("spatial_features")
- communication_rate = output.get("communication_rate")
- if not isinstance(spatial_features, torch.Tensor):
- raise TypeError('Where2Comm output field "spatial_features" must be a tensor')
- if not isinstance(communication_rate, torch.Tensor):
- raise TypeError('Where2Comm output field "communication_rate" must be a tensor')
- return (
- spatial_features.detach().cpu().numpy(),
- float(communication_rate.detach().cpu().item()),
- )
-
- def _build_wire_payload(
- self,
- dataset: Where2CommCommunicationDataset,
- inference_input: InferenceInput,
- metadata: PoseFrameMetadata,
- ) -> Where2CommWirePayload:
- """
- Build one sender's learned-feature payload.
-
- Parameters
- ----------
- dataset : Where2CommCommunicationDataset
- Dataset providing the configured preprocessor.
- inference_input : InferenceInput
- Sender-local preprocessor output.
- metadata : PoseFrameMetadata
- Sender pose and capture frame.
-
- Returns
- -------
- Where2CommWirePayload
- Masked learned features and sender metadata.
- """
- spatial_features, communication_rate = self._encode(
- dataset,
- inference_input,
- apply_communication_mask=True,
- )
- return Where2CommWirePayload(
- inference_input=Where2CommFeatureInput(
- spatial_features=spatial_features,
- communication_rate=communication_rate,
- ),
- metadata=metadata,
- )
-
- def encode_local_intermediate_input(
- self,
- dataset: Any,
- inference_input: Any,
- ) -> dict[str, Any]:
- """
- Encode the receiver's own feature without an over-the-air mask.
-
- Parameters
- ----------
- dataset : Any
- Intermediate-fusion dataset providing the preprocessor.
- inference_input : Any
- Receiver-local typed preprocessor output.
-
- Returns
- -------
- dict[str, Any]
- Receiver-local learned feature input.
- """
- if not isinstance(dataset, Where2CommCommunicationDataset):
- raise TypeError("Where2Comm requires an intermediate-fusion communication dataset")
- spatial_features, communication_rate = self._encode(
- dataset,
- inference_input,
- apply_communication_mask=False,
- )
- return {
- "spatial_features": spatial_features,
- "communication_rate": communication_rate,
- }
-
- def decode_received_payload(
- self,
- payload: object,
- receiver_pose: list[float],
- ) -> dict[str, Any]:
- """
- Decode one successfully delivered Where2Comm feature payload.
-
- Parameters
- ----------
- payload : object
- Deserialized Where2Comm payload delivered over V2X.
- receiver_pose : list[float]
- Receiver pose reserved for later receiver-side feature alignment.
-
- Returns
- -------
- dict[str, Any]
- Learned feature map and communication rate.
-
- Raises
- ------
- TypeError
- If a payload belonging to another contract is received.
- """
- del receiver_pose
- if not isinstance(payload, Where2CommWirePayload):
- raise TypeError(f"Expected Where2CommWirePayload, got {type(payload).__name__}")
- return {
- "spatial_features": np.array(
- payload.inference_input.spatial_features,
- copy=True,
- ),
- "communication_rate": payload.inference_input.communication_rate,
- "metadata": payload.metadata,
- }
diff --git a/OpenCOOD/opencood/models/communication_adapters/wire_payload.py b/OpenCOOD/opencood/models/communication_adapters/wire_payload.py
deleted file mode 100644
index 56ee71e9..00000000
--- a/OpenCOOD/opencood/models/communication_adapters/wire_payload.py
+++ /dev/null
@@ -1,104 +0,0 @@
-"""Typed cooperative perception payloads exchanged over V2X."""
-
-from dataclasses import dataclass
-from typing import Any
-
-import numpy.typing as npt
-
-from opencood.models.communication_adapters.inference_input import (
- DetectionInferenceInput,
- FpvrcnnAgentInferenceInput,
- MultiScaleFeatureInput,
- SpatialFeatureInput,
- Where2CommFeatureInput,
-)
-from opencood.models.communication_adapters.inference_metadata import (
- IntermediateMetadata,
- PoseFrameMetadata,
-)
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class EarlyFusionWirePayload:
- """
- Sender-local LiDAR data exchanged for early fusion.
-
- Attributes
- ----------
- lidar_points : numpy.typing.NDArray[Any]
- LiDAR points expressed in the sender coordinate frame.
- metadata : PoseFrameMetadata
- Sender pose and capture frame used for receiver-side projection.
- """
-
- lidar_points: npt.NDArray[Any]
- metadata: PoseFrameMetadata
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class LateFusionWirePayload:
- """
- Sender-side detections exchanged for late fusion.
-
- Attributes
- ----------
- detections : DetectionInferenceInput
- Bounding-box candidates and confidence scores produced by the sender.
- metadata : PoseFrameMetadata
- Pose and capture frame required for late-fusion post-processing.
- """
-
- detections: DetectionInferenceInput
- metadata: PoseFrameMetadata
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class FpvrcnnWirePayload:
- """
- Agent-side FPV-RCNN output exchanged over V2X.
-
- Attributes
- ----------
- inference_input : FpvrcnnAgentInferenceInput
- Stage-one proposals and VSA keypoint features computed by the sender.
- metadata : PoseFrameMetadata
- Sender pose and capture frame used to transform the output at the
- receiver.
- """
-
- inference_input: FpvrcnnAgentInferenceInput
- metadata: PoseFrameMetadata
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class Where2CommWirePayload:
- """
- Sender-side learned features exchanged by Where2Comm.
-
- Attributes
- ----------
- inference_input : Where2CommFeatureInput
- Masked feature map and its effective communication rate.
- metadata : PoseFrameMetadata
- Sender pose and capture frame used by the fusion pipeline.
- """
-
- inference_input: Where2CommFeatureInput
- metadata: PoseFrameMetadata
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class IntermediateFeatureWirePayload:
- """
- Learned features exchanged by an intermediate-fusion model.
-
- Attributes
- ----------
- inference_input : SpatialFeatureInput | MultiScaleFeatureInput
- Model-specific learned features produced by the sender.
- metadata : IntermediateMetadata | None
- Metadata required by the receiver-side fusion implementation.
- """
-
- inference_input: SpatialFeatureInput | MultiScaleFeatureInput
- metadata: IntermediateMetadata | None
diff --git a/OpenCOOD/opencood/models/fpvrcnn.py b/OpenCOOD/opencood/models/fpvrcnn.py
deleted file mode 100644
index f3e71b3c..00000000
--- a/OpenCOOD/opencood/models/fpvrcnn.py
+++ /dev/null
@@ -1,230 +0,0 @@
-import numpy as np
-import torch
-from torch import nn
-
-from opencood.models.communication_adapters.fpvrcnn import FpvrcnnCommunicationAdapter
-from opencood.models.sub_modules.mean_vfe import MeanVFE
-from opencood.models.sub_modules.sparse_backbone_3d import VoxelBackBone8x
-from opencood.models.sub_modules.height_compression import HeightCompression
-from opencood.models.sub_modules.cia_ssd_utils import SSFA, Head
-from opencood.models.sub_modules.vsa import VoxelSetAbstraction
-from opencood.models.sub_modules.roi_head import RoIHead
-from opencood.models.sub_modules.matcher import Matcher
-from opencood.data_utils.post_processor.fpvrcnn_postprocessor import FpvrcnnPostprocessor
-
-
-class FPVRCNN(nn.Module):
- # The model declares its own wire boundary; neither the manager nor the
- # dataset needs to infer it from the selected fusion dataset.
- communication_adapter_class = FpvrcnnCommunicationAdapter
-
- def __init__(self, args):
- super(FPVRCNN, self).__init__()
- lidar_range = np.array(args["lidar_range"])
- grid_size = np.round((lidar_range[3:6] - lidar_range[:3]) / np.array(args["voxel_size"])).astype(np.int64)
- self.vfe = MeanVFE(args["mean_vfe"], args["mean_vfe"]["num_point_features"])
- self.spconv_block = VoxelBackBone8x(args["spconv"], input_channels=args["spconv"]["num_features_in"], grid_size=grid_size)
- self.map_to_bev = HeightCompression(args["map2bev"])
- self.ssfa = SSFA(args["ssfa"])
- self.head = Head(**args["head"])
- self.post_processor = FpvrcnnPostprocessor(args["post_processer"], train=True)
- self.vsa = VoxelSetAbstraction(args["vsa"], args["voxel_size"], args["lidar_range"], num_bev_features=128, num_rawpoint_features=3)
- self.matcher = Matcher(args["matcher"], args["lidar_range"])
- self.roi_head = RoIHead(args["roi_head"])
- self.train_stage2 = args["activate_stage2"]
-
- def _forward_stage_one(self, batch_dict):
- """
- Run the per-agent backbone and proposal head.
-
- Parameters
- ----------
- batch_dict : dict
- Collated voxel input for one or more independent agents.
-
- Returns
- -------
- tuple[dict, list[torch.Tensor] | None, list[torch.Tensor] | None]
- Updated model state, proposal boxes, and proposal scores.
- """
- voxel_features = batch_dict["processed_lidar"]["voxel_features"]
- voxel_coords = batch_dict["processed_lidar"]["voxel_coords"]
- voxel_num_points = batch_dict["processed_lidar"]["voxel_num_points"]
-
- # save memory
- batch_dict.pop("processed_lidar")
- batch_dict.update(
- {
- "voxel_features": voxel_features,
- "voxel_coords": voxel_coords,
- "voxel_num_points": voxel_num_points,
- "batch_size": int(batch_dict["record_len"].sum()),
- }
- )
-
- batch_dict = self.vfe(batch_dict)
- batch_dict = self.spconv_block(batch_dict)
- batch_dict = self.map_to_bev(batch_dict)
-
- out = self.ssfa(batch_dict["spatial_features"])
- batch_dict["preds_dict_stage1"] = self.head(out)
-
- data_dict, output_dict = {}, {}
- data_dict["ego"], output_dict["ego"] = batch_dict, batch_dict
-
- pred_box3d_list, scores_list = self.post_processor.post_process(data_dict, output_dict, stage1=True)
- batch_dict["det_boxes"] = pred_box3d_list
- batch_dict["det_scores"] = scores_list
-
- return batch_dict, pred_box3d_list, scores_list
-
- def encode_agent(self, batch_dict):
- """
- Build the FPV-RCNN representation transmitted by one agent.
-
- Parameters
- ----------
- batch_dict : dict
- Single-agent voxel input, raw LiDAR points, and anchors.
-
- Returns
- -------
- dict[str, torch.Tensor]
- Stage-one proposals and VSA keypoint coordinates/features in the
- sender coordinate frame.
-
- Raises
- ------
- RuntimeError
- If cooperative stage two is disabled in the model configuration.
- """
- if not self.train_stage2:
- raise RuntimeError("Distributed FPV-RCNN inference requires activate_stage2=true")
- batch_dict, proposals, proposal_scores = self._forward_stage_one(batch_dict)
- if proposals is None or proposal_scores is None:
- device = batch_dict["voxel_features"].device
- return {
- "proposals": torch.empty((0, 7), device=device),
- "proposal_scores": torch.empty((0,), device=device),
- "point_coords": torch.empty((0, 4), device=device),
- "point_features": torch.empty((0, self.vsa.num_point_features), device=device),
- }
-
- batch_dict = self.vsa(batch_dict)
- return {
- "proposals": proposals[0],
- "proposal_scores": proposal_scores[0],
- "point_coords": batch_dict["point_coords"][0],
- "point_features": batch_dict["point_features"][0],
- }
-
- def _forward_distributed(self, batch_dict):
- """
- Fuse local FPV-RCNN output with delivered agent outputs.
-
- Parameters
- ----------
- batch_dict : dict
- Ego input containing ``remote_agent_outputs`` decoded from wire
- payloads.
-
- Returns
- -------
- dict
- Cooperative stage-two model output.
- """
- remote_outputs = batch_dict.pop("remote_agent_outputs")
- local_output = self.encode_agent(batch_dict)
- agent_outputs = [local_output, *remote_outputs]
-
- batch_dict["det_boxes"] = [output["proposals"] for output in agent_outputs]
- batch_dict["det_scores"] = [output["proposal_scores"] for output in agent_outputs]
- batch_dict["point_coords"] = [output["point_coords"] for output in agent_outputs]
- batch_dict["point_features"] = [output["point_features"] for output in agent_outputs]
- batch_dict["record_len"] = torch.tensor(
- [len(agent_outputs)],
- dtype=torch.long,
- device=batch_dict["voxel_features"].device,
- )
-
- if not any(len(boxes) > 0 for boxes in batch_dict["det_boxes"]):
- batch_dict["fpvrcnn_out"] = None
- return batch_dict
-
- batch_dict = self.matcher(batch_dict)
- return self.roi_head(batch_dict)
-
- @staticmethod
- def _transform_agent_outputs(batch_dict):
- """
- Transform local proposals and keypoints into each scene's ego frame.
-
- Parameters
- ----------
- batch_dict : dict
- Model state containing per-agent detections, keypoints, and one
- ``agent_to_ego`` matrix per flattened agent.
-
- Returns
- -------
- dict
- Model state with proposal centers, headings, and keypoint
- coordinates expressed in their corresponding ego frames.
- """
- transformations = batch_dict.pop("agent_to_ego")
- if len(transformations) != len(batch_dict["det_boxes"]) or len(transformations) != len(batch_dict["point_coords"]):
- raise ValueError("Every FPV-RCNN agent output requires one agent-to-ego transformation")
- transformed_boxes = []
- transformed_points = []
-
- for boxes, points, transformation in zip(
- batch_dict["det_boxes"],
- batch_dict["point_coords"],
- transformations,
- ):
- boxes = boxes.clone()
- if len(boxes) > 0:
- box_transformation = transformation.to(dtype=boxes.dtype)
- homogeneous_centers = torch.cat(
- [boxes[:, :3], torch.ones((len(boxes), 1), dtype=boxes.dtype, device=boxes.device)],
- dim=1,
- )
- boxes[:, :3] = (box_transformation @ homogeneous_centers.T).T[:, :3]
- boxes[:, 6] += torch.atan2(box_transformation[1, 0], box_transformation[0, 0])
- transformed_boxes.append(boxes)
-
- points = points.clone()
- if len(points) > 0:
- point_transformation = transformation.to(dtype=points.dtype)
- homogeneous_points = torch.cat(
- [points[:, :3], torch.ones((len(points), 1), dtype=points.dtype, device=points.device)],
- dim=1,
- )
- points[:, :3] = (point_transformation @ homogeneous_points.T).T[:, :3]
- transformed_points.append(points)
-
- batch_dict["det_boxes"] = transformed_boxes
- batch_dict["point_coords"] = transformed_points
- return batch_dict
-
- def forward(self, batch_dict):
- if "remote_agent_outputs" in batch_dict:
- return self._forward_distributed(batch_dict)
-
- batch_dict, pred_box3d_list, _ = self._forward_stage_one(batch_dict)
-
- if pred_box3d_list is not None and self.train_stage2:
- batch_dict = self.vsa(batch_dict)
- if "agent_to_ego" in batch_dict:
- batch_dict = self._transform_agent_outputs(batch_dict)
- batch_dict = self.matcher(batch_dict)
- batch_dict = self.roi_head(batch_dict)
- if pred_box3d_list is None:
- batch_dict["fpvrcnn_out"] = None
-
- return batch_dict
-
-
-if __name__ == "__main__":
- model = SSFA(None)
- print(model)
diff --git a/OpenCOOD/opencood/models/fuse_modules/V2VAM.py b/OpenCOOD/opencood/models/fuse_modules/V2VAM.py
deleted file mode 100644
index 98ea7b29..00000000
--- a/OpenCOOD/opencood/models/fuse_modules/V2VAM.py
+++ /dev/null
@@ -1,106 +0,0 @@
-import torch
-import torch.nn as nn
-
-
-from torch.nn import Softmax
-
-
-class V2V_AttFusion(nn.Module):
- def __init__(self, feature_dim):
- super(V2V_AttFusion, self).__init__()
-
- self.cov_att = nn.Sequential(
- nn.Conv2d(in_channels=feature_dim, out_channels=feature_dim, kernel_size=3, padding=1),
- nn.BatchNorm2d(feature_dim, eps=1e-5, momentum=0.01, affine=True),
- nn.ReLU(),
- )
-
- self.CCNet = CrissCrossAttention(feature_dim)
-
- def forward(self, x, record_len):
- split_x = self.regroup(x, record_len) # x =[5, 64, 100, 352], record_len=[3,2]
-
- out = []
- att = []
- for xx in split_x: # split_x[0] [num_car, C, W, H]
- """ CCNet: Criss-Cross Attention Module: attention for ego vehicle feature + cav feature """
-
- ego_q, _, _ = xx[0:1], xx[0:1], xx[0:1] # 2 -> ego_k, 3 -> ego_v
- for i in range(len(xx[:, 0, 0, 0])):
- att_vehicle = self.CCNet(ego_q, xx[i : i + 1], xx[i : i + 1])
- att.append(att_vehicle)
-
- pooling_max = torch.max(torch.cat(att, dim=0), dim=0, keepdim=True)[0]
- pooling_ave = torch.mean(torch.cat(att, dim=0), dim=0, keepdim=True)[0]
-
- fuse_fea = pooling_max + pooling_ave
-
- fuse_att = fuse_fea
- fuse_att = self.cov_att(fuse_att)
-
- out.append(fuse_att) # [[1, 64, 100, 352], [1, 64, 100, 352]]
- # torch.cuda.empty_cache()
-
- return torch.cat(out, dim=0) # [2, 64, 100, 352]
-
- def regroup(self, x, record_len):
- cum_sum_len = torch.cumsum(record_len, dim=0)
- split_x = torch.tensor_split(x, cum_sum_len[:-1].cpu())
-
- return split_x
-
-
-def INF(B, H, W):
- return -torch.diag(torch.tensor(float("inf")).cuda().repeat(H), 0).unsqueeze(0).repeat(B * W, 1, 1)
-
-
-class CrissCrossAttention(nn.Module):
- """Criss-Cross Attention Module
-
- reference: https://github.com/speedinghzl/CCNet
-
- """
-
- def __init__(self, in_dim):
- super(CrissCrossAttention, self).__init__()
-
- self.query_conv = nn.Sequential(
- nn.Conv2d(in_channels=in_dim, out_channels=in_dim, kernel_size=1), nn.BatchNorm2d(in_dim, eps=1e-5, momentum=0.01, affine=True), nn.ReLU()
- )
- self.key_conv = nn.Sequential(
- nn.Conv2d(in_channels=in_dim, out_channels=in_dim, kernel_size=1), nn.BatchNorm2d(in_dim, eps=1e-5, momentum=0.01, affine=True), nn.ReLU()
- )
- self.value_conv = nn.Sequential(
- nn.Conv2d(in_channels=in_dim, out_channels=in_dim, kernel_size=1), nn.BatchNorm2d(in_dim, eps=1e-5, momentum=0.01, affine=True), nn.ReLU()
- )
-
- self.softmax = Softmax(dim=3)
- self.INF = INF
- self.gamma = nn.Parameter(torch.zeros(1))
-
- def forward(self, query, key, value):
- m_batchsize, _, height, width = query.size()
-
- proj_query = self.query_conv(query)
- proj_query_H = proj_query.permute(0, 3, 1, 2).contiguous().view(m_batchsize * width, -1, height).permute(0, 2, 1)
- proj_query_W = proj_query.permute(0, 2, 1, 3).contiguous().view(m_batchsize * height, -1, width).permute(0, 2, 1)
-
- proj_key = self.key_conv(key)
- proj_key_H = proj_key.permute(0, 3, 1, 2).contiguous().view(m_batchsize * width, -1, height)
- proj_key_W = proj_key.permute(0, 2, 1, 3).contiguous().view(m_batchsize * height, -1, width)
-
- proj_value = self.value_conv(value)
- proj_value_H = proj_value.permute(0, 3, 1, 2).contiguous().view(m_batchsize * width, -1, height)
- proj_value_W = proj_value.permute(0, 2, 1, 3).contiguous().view(m_batchsize * height, -1, width)
- energy_H = (
- (torch.bmm(proj_query_H, proj_key_H) + self.INF(m_batchsize, height, width)).view(m_batchsize, width, height, height).permute(0, 2, 1, 3)
- )
- energy_W = torch.bmm(proj_query_W, proj_key_W).view(m_batchsize, height, width, width)
- concate = self.softmax(torch.cat([energy_H, energy_W], 3))
-
- att_H = concate[:, :, :, 0:height].permute(0, 2, 1, 3).contiguous().view(m_batchsize * width, height, height)
- att_W = concate[:, :, :, height : height + width].contiguous().view(m_batchsize * height, width, width)
- out_H = torch.bmm(proj_value_H, att_H.permute(0, 2, 1)).view(m_batchsize, width, -1, height).permute(0, 2, 3, 1)
- out_W = torch.bmm(proj_value_W, att_W.permute(0, 2, 1)).view(m_batchsize, height, -1, width).permute(0, 2, 1, 3)
-
- return self.gamma * (out_H + out_W) + value
diff --git a/OpenCOOD/opencood/models/fuse_modules/__init__.py b/OpenCOOD/opencood/models/fuse_modules/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/OpenCOOD/opencood/models/fuse_modules/coalign_fuse.py b/OpenCOOD/opencood/models/fuse_modules/coalign_fuse.py
deleted file mode 100644
index 9ef0e65b..00000000
--- a/OpenCOOD/opencood/models/fuse_modules/coalign_fuse.py
+++ /dev/null
@@ -1,76 +0,0 @@
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-from opencood.models.fuse_modules.self_attn import ScaledDotProductAttention
-
-
-def regroup(x, record_len):
- cum_sum_len = torch.cumsum(record_len, dim=0)
- split_x = torch.tensor_split(x, cum_sum_len[:-1].cpu())
- return split_x
-
-
-def normalize_pairwise_tfm(pairwise_t_matrix, H, W, discrete_ratio, downsample_rate=1):
- """
- normalize the pairwise transformation matrix to affine matrix need by torch.nn.functional.affine_grid()
-
- Parameters
- ----------
- pairwise_t_matrix: torch.tensor
- [B, L, L, 4, 4], B batchsize, L max_cav
- H: num.
- Feature map height
- W: num.
- Feature map width
- discrete_ratio * downsample_rate: num.
- One pixel on the feature map corresponds to the actual physical distance
-
- Returns
- -------
- normalized_affine_matrix: torch.tensor
- [B, L, L, 2, 3], B batchsize, L max_cav
- """
- pairwise_t_matrix = pairwise_t_matrix[:, :, :, [0, 1], :][:, :, :, :, [0, 1, 3]] # [B, L, L, 2, 3]
- pairwise_t_matrix[..., 0, 1] = pairwise_t_matrix[..., 0, 1] * H / W
- pairwise_t_matrix[..., 1, 0] = pairwise_t_matrix[..., 1, 0] * W / H
- pairwise_t_matrix[..., 0, 2] = pairwise_t_matrix[..., 0, 2] / (downsample_rate * discrete_ratio * W) * 2
- pairwise_t_matrix[..., 1, 2] = pairwise_t_matrix[..., 1, 2] / (downsample_rate * discrete_ratio * H) * 2
-
- normalized_affine_matrix = pairwise_t_matrix
-
- return normalized_affine_matrix
-
-
-def warp_affine_simple(src, M, dsize, mode="bilinear", padding_mode="zeros", align_corners=False):
- """ """
- B, C, H, W = src.size()
- grid = F.affine_grid(M, [B, C, dsize[0], dsize[1]], align_corners=align_corners).to(src)
- return F.grid_sample(src, grid, align_corners=align_corners)
-
-
-class Att_w_Warp(nn.Module):
- def __init__(self, feature_dims):
- super(Att_w_Warp, self).__init__()
- self.att = ScaledDotProductAttention(feature_dims)
-
- def forward(self, xx, record_len, normalized_affine_matrix):
- _, C, H, W = xx.shape
- B, L = normalized_affine_matrix.shape[:2]
- split_x = regroup(xx, record_len)
- batch_node_features = split_x
- out = []
- # iterate each batch
- for b in range(B):
- N = record_len[b]
- t_matrix = normalized_affine_matrix[b][:N, :N, :, :]
- # update each node i
- i = 0 # ego
- x = warp_affine_simple(batch_node_features[b], t_matrix[i, :, :, :], (H, W))
- cav_num = x.shape[0]
- x = x.view(cav_num, C, -1).permute(2, 0, 1) # (H*W, cav_num, C), perform self attention on each pixel.
- h = self.att(x, x, x)
- h = h.permute(1, 2, 0).view(cav_num, C, H, W)[0, ...] # C, W, H before
- out.append(h)
-
- out = torch.stack(out)
- return out
diff --git a/OpenCOOD/opencood/models/fuse_modules/f_cooper_fuse.py b/OpenCOOD/opencood/models/fuse_modules/f_cooper_fuse.py
deleted file mode 100644
index d8bc68b7..00000000
--- a/OpenCOOD/opencood/models/fuse_modules/f_cooper_fuse.py
+++ /dev/null
@@ -1,26 +0,0 @@
-"""
-Implementation of F-cooper maxout fusing.
-"""
-
-import torch
-import torch.nn as nn
-
-
-class SpatialFusion(nn.Module):
- def __init__(self):
- super(SpatialFusion, self).__init__()
-
- def regroup(self, x, record_len):
- cum_sum_len = torch.cumsum(record_len, dim=0)
- split_x = torch.tensor_split(x, cum_sum_len[:-1].cpu())
- return split_x
-
- def forward(self, x, record_len):
- # x: B, C, H, W, split x:[(B1, C, W, H), (B2, C, W, H)]
- split_x = self.regroup(x, record_len)
- out = []
-
- for xx in split_x:
- xx = torch.max(xx, dim=0, keepdim=True)[0]
- out.append(xx)
- return torch.cat(out, dim=0)
diff --git a/OpenCOOD/opencood/models/fuse_modules/fuse_utils.py b/OpenCOOD/opencood/models/fuse_modules/fuse_utils.py
deleted file mode 100644
index 8a00f196..00000000
--- a/OpenCOOD/opencood/models/fuse_modules/fuse_utils.py
+++ /dev/null
@@ -1,54 +0,0 @@
-import torch
-import numpy as np
-
-from einops import rearrange
-from opencood.utils.common_utils import torch_tensor_to_numpy
-
-
-def regroup(dense_feature, record_len, max_len):
- """
- Regroup the data based on the record_len.
-
- Parameters
- ----------
- dense_feature : torch.Tensor
- N, C, H, W
- record_len : list
- [sample1_len, sample2_len, ...]
- max_len : int
- Maximum cav number
-
- Returns
- -------
- regroup_feature : torch.Tensor
- B, L, C, H, W
- """
- cum_sum_len = list(np.cumsum(torch_tensor_to_numpy(record_len)))
- split_features = torch.tensor_split(dense_feature, cum_sum_len[:-1])
- regroup_features = []
- mask = []
-
- for split_feature in split_features:
- # M, C, H, W
- feature_shape = split_feature.shape
-
- # the maximum M is 5 as most 5 cavs
- padding_len = max_len - feature_shape[0]
- mask.append([1] * feature_shape[0] + [0] * padding_len)
-
- padding_tensor = torch.zeros(padding_len, feature_shape[1], feature_shape[2], feature_shape[3])
- padding_tensor = padding_tensor.to(split_feature.device)
-
- split_feature = torch.cat([split_feature, padding_tensor], dim=0)
-
- # 1, 5C, H, W
- split_feature = split_feature.view(-1, feature_shape[2], feature_shape[3]).unsqueeze(0)
- regroup_features.append(split_feature)
-
- # B, 5C, H, W
- regroup_features = torch.cat(regroup_features, dim=0)
- # B, L, C, H, W
- regroup_features = rearrange(regroup_features, "b (l c) h w -> b l c h w", l=max_len)
- mask = torch.from_numpy(np.array(mask)).to(regroup_features.device)
-
- return regroup_features, mask
diff --git a/OpenCOOD/opencood/models/fuse_modules/hmsa.py b/OpenCOOD/opencood/models/fuse_modules/hmsa.py
deleted file mode 100644
index 4922a73a..00000000
--- a/OpenCOOD/opencood/models/fuse_modules/hmsa.py
+++ /dev/null
@@ -1,135 +0,0 @@
-import torch
-from torch import nn
-
-from einops import rearrange
-
-
-class HGTCavAttention(nn.Module):
- def __init__(self, dim, heads, num_types=2, num_relations=4, dim_head=64, dropout=0.1):
- super().__init__()
- inner_dim = heads * dim_head
-
- self.heads = heads
- self.scale = dim_head**-0.5
- self.num_types = num_types
-
- self.attend = nn.Softmax(dim=-1)
- self.drop_out = nn.Dropout(dropout)
- self.k_linears = nn.ModuleList()
- self.q_linears = nn.ModuleList()
- self.v_linears = nn.ModuleList()
- self.a_linears = nn.ModuleList()
- self.norms = nn.ModuleList() # noqa: DC05
- for t in range(num_types):
- self.k_linears.append(nn.Linear(dim, inner_dim))
- self.q_linears.append(nn.Linear(dim, inner_dim))
- self.v_linears.append(nn.Linear(dim, inner_dim))
- self.a_linears.append(nn.Linear(inner_dim, dim))
-
- self.relation_att = nn.Parameter(torch.Tensor(num_relations, heads, dim_head, dim_head))
- self.relation_msg = nn.Parameter(torch.Tensor(num_relations, heads, dim_head, dim_head))
-
- torch.nn.init.xavier_uniform(self.relation_att)
- torch.nn.init.xavier_uniform(self.relation_msg)
-
- def to_qkv(self, x, types):
- # x: (B,H,W,L,C)
- # types: (B,L)
- q_batch = []
- k_batch = []
- v_batch = []
-
- for b in range(x.shape[0]):
- q_list = []
- k_list = []
- v_list = []
-
- for i in range(x.shape[-2]):
- # (H,W,1,C)
- q_list.append(self.q_linears[types[b, i]](x[b, :, :, i, :].unsqueeze(2)))
- k_list.append(self.k_linears[types[b, i]](x[b, :, :, i, :].unsqueeze(2)))
- v_list.append(self.v_linears[types[b, i]](x[b, :, :, i, :].unsqueeze(2)))
- # (1,H,W,L,C)
- q_batch.append(torch.cat(q_list, dim=2).unsqueeze(0))
- k_batch.append(torch.cat(k_list, dim=2).unsqueeze(0))
- v_batch.append(torch.cat(v_list, dim=2).unsqueeze(0))
- # (B,H,W,L,C)
- q = torch.cat(q_batch, dim=0)
- k = torch.cat(k_batch, dim=0)
- v = torch.cat(v_batch, dim=0)
- return q, k, v
-
- def get_relation_type_index(self, type1, type2):
- return type1 * self.num_types + type2
-
- def get_hetero_edge_weights(self, x, types):
- w_att_batch = []
- w_msg_batch = []
-
- for b in range(x.shape[0]):
- w_att_list = []
- w_msg_list = []
-
- for i in range(x.shape[-2]):
- w_att_i_list = []
- w_msg_i_list = []
-
- for j in range(x.shape[-2]):
- e_type = self.get_relation_type_index(types[b, i], types[b, j])
- w_att_i_list.append(self.relation_att[e_type].unsqueeze(0))
- w_msg_i_list.append(self.relation_msg[e_type].unsqueeze(0))
- w_att_list.append(torch.cat(w_att_i_list, dim=0).unsqueeze(0))
- w_msg_list.append(torch.cat(w_msg_i_list, dim=0).unsqueeze(0))
-
- w_att_batch.append(torch.cat(w_att_list, dim=0).unsqueeze(0))
- w_msg_batch.append(torch.cat(w_msg_list, dim=0).unsqueeze(0))
-
- # (B,M,L,L,C_head,C_head)
- w_att = torch.cat(w_att_batch, dim=0).permute(0, 3, 1, 2, 4, 5)
- w_msg = torch.cat(w_msg_batch, dim=0).permute(0, 3, 1, 2, 4, 5)
- return w_att, w_msg
-
- def to_out(self, x, types):
- out_batch = []
- for b in range(x.shape[0]):
- out_list = []
- for i in range(x.shape[-2]):
- out_list.append(self.a_linears[types[b, i]](x[b, :, :, i, :].unsqueeze(2)))
- out_batch.append(torch.cat(out_list, dim=2).unsqueeze(0))
- out = torch.cat(out_batch, dim=0)
- return out
-
- def forward(self, x, mask, prior_encoding):
- # x: (B, L, H, W, C) -> (B, H, W, L, C)
- # mask: (B, H, W, L, 1)
- # prior_encoding: (B,L,H,W,3)
- x = x.permute(0, 2, 3, 1, 4)
- # mask: (B, 1, H, W, L, 1)
- mask = mask.unsqueeze(1)
- # (B,L)
- _, dts, types = [itm.squeeze(-1) for itm in prior_encoding[:, :, 0, 0, :].split([1, 1, 1], dim=-1)]
- types = types.to(torch.int)
- dts = dts.to(torch.int)
- qkv = self.to_qkv(x, types)
- # (B,M,L,L,C_head,C_head)
- w_att, w_msg = self.get_hetero_edge_weights(x, types)
-
- # q: (B, M, H, W, L, C)
- q, k, v = map(lambda t: rearrange(t, "b h w l (m c) -> b m h w l c", m=self.heads), (qkv))
- # attention, (B, M, H, W, L, L)
- att_map = torch.einsum("b m h w i p, b m i j p q, bm h w j q -> b m h w i j", [q, w_att, k]) * self.scale
- # add mask
- att_map = att_map.masked_fill(mask == 0, -float("inf"))
- # softmax
- att_map = self.attend(att_map)
-
- # out:(B, M, H, W, L, C_head)
- v_msg = torch.einsum("b m i j p c, b m h w j p -> b m h w i j c", w_msg, v)
- out = torch.einsum("b m h w i j, b m h w i j c -> b m h w i c", att_map, v_msg)
-
- out = rearrange(out, "b m h w l c -> b h w l (m c)", m=self.heads)
- out = self.to_out(out, types)
- out = self.drop_out(out)
- # (B L H W C)
- out = out.permute(0, 3, 1, 2, 4)
- return out
diff --git a/OpenCOOD/opencood/models/fuse_modules/mswin.py b/OpenCOOD/opencood/models/fuse_modules/mswin.py
deleted file mode 100644
index a2a41c1b..00000000
--- a/OpenCOOD/opencood/models/fuse_modules/mswin.py
+++ /dev/null
@@ -1,116 +0,0 @@
-"""
-Multi-scale window transformer
-"""
-
-import torch
-import torch.nn as nn
-import numpy as np
-
-from einops import rearrange
-from opencood.models.sub_modules.split_attn import SplitAttn
-
-
-def get_relative_distances(window_size):
- indices = torch.tensor(np.array([[x, y] for x in range(window_size) for y in range(window_size)]))
- distances = indices[None, :, :] - indices[:, None, :]
- return distances
-
-
-class BaseWindowAttention(nn.Module):
- def __init__(self, dim, heads, dim_head, drop_out, window_size, relative_pos_embedding):
- super().__init__()
- inner_dim = dim_head * heads
-
- self.heads = heads
- self.scale = dim_head**-0.5
- self.window_size = window_size
- self.relative_pos_embedding = relative_pos_embedding
-
- self.to_qkv = nn.Linear(dim, inner_dim * 3, bias=False)
-
- if self.relative_pos_embedding:
- self.relative_indices = get_relative_distances(window_size) + window_size - 1
- self.pos_embedding = nn.Parameter(torch.randn(2 * window_size - 1, 2 * window_size - 1))
- else:
- self.pos_embedding = nn.Parameter(torch.randn(window_size**2, window_size**2))
-
- self.to_out = nn.Sequential(nn.Linear(inner_dim, dim), nn.Dropout(drop_out))
-
- def forward(self, x):
- _, _, h, w, _, m = *x.shape, self.heads # 1 -> b, 2 -> length, 5 -> c
-
- qkv = self.to_qkv(x).chunk(3, dim=-1)
- new_h = h // self.window_size
- new_w = w // self.window_size
-
- # q : (b, l, m, new_h*new_w, window_size^2, c_head)
- q, k, v = map(
- lambda t: rearrange(
- t, "b l (new_h w_h) (new_w w_w) (m c) -> b l m (new_h new_w) (w_h w_w) c", m=m, w_h=self.window_size, w_w=self.window_size
- ),
- qkv,
- )
- # b l m h window_size window_size
- dots = (
- torch.einsum(
- "b l m h i c, b l m h j c -> b l m h i j",
- q,
- k,
- )
- * self.scale
- )
- # consider prior knowledge of the local window
- if self.relative_pos_embedding:
- dots += self.pos_embedding[self.relative_indices[:, :, 0], self.relative_indices[:, :, 1]]
- else:
- dots += self.pos_embedding
-
- attn = dots.softmax(dim=-1)
-
- out = torch.einsum("b l m h i j, b l m h j c -> b l m h i c", attn, v)
- # b l h w c
- out = rearrange(
- out,
- "b l m (new_h new_w) (w_h w_w) c -> b l (new_h w_h) (new_w w_w) (m c)",
- m=self.heads,
- w_h=self.window_size,
- w_w=self.window_size,
- new_w=new_w,
- new_h=new_h,
- )
- out = self.to_out(out)
-
- return out
-
-
-class PyramidWindowAttention(nn.Module):
- def __init__(self, dim, heads, dim_heads, drop_out, window_size, relative_pos_embedding, fuse_method="naive"):
- super().__init__()
-
- assert isinstance(window_size, list)
- assert isinstance(heads, list)
- assert isinstance(dim_heads, list)
- assert len(dim_heads) == len(heads)
-
- self.pwmsa = nn.ModuleList([])
-
- for head, dim_head, ws in zip(heads, dim_heads, window_size):
- self.pwmsa.append(BaseWindowAttention(dim, head, dim_head, drop_out, ws, relative_pos_embedding))
- self.fuse_mehod = fuse_method
- if fuse_method == "split_attn":
- self.split_attn = SplitAttn(256)
-
- def forward(self, x):
- output = None
- # naive fusion will just sum up all window attention output and do a
- # mean
- if self.fuse_mehod == "naive":
- for wmsa in self.pwmsa:
- output = wmsa(x) if output is None else output + wmsa(x)
- return output / len(self.pwmsa)
-
- elif self.fuse_mehod == "split_attn":
- window_list = []
- for wmsa in self.pwmsa:
- window_list.append(wmsa(x))
- return self.split_attn(window_list)
diff --git a/OpenCOOD/opencood/models/fuse_modules/self_attn.py b/OpenCOOD/opencood/models/fuse_modules/self_attn.py
deleted file mode 100644
index 2133ed93..00000000
--- a/OpenCOOD/opencood/models/fuse_modules/self_attn.py
+++ /dev/null
@@ -1,66 +0,0 @@
-import numpy as np
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-
-
-class ScaledDotProductAttention(nn.Module):
- """
- Scaled Dot-Product Attention proposed in "Attention Is All You Need"
- Compute the dot products of the query with all keys, divide each by sqrt(dim),
- and apply a softmax function to obtain the weights on the values
- Args: dim, mask
- dim (int): dimention of attention
- mask (torch.Tensor): tensor containing indices to be masked
- Inputs: query, key, value, mask
- - **query** (batch, q_len, d_model): tensor containing projection
- vector for decoder.
- - **key** (batch, k_len, d_model): tensor containing projection
- vector for encoder.
- - **value** (batch, v_len, d_model): tensor containing features of the
- encoded input sequence.
- - **mask** (-): tensor containing indices to be masked
- Returns: context, attn
- - **context**: tensor containing the context vector from
- attention mechanism.
- - **attn**: tensor containing the attention (alignment) from the
- encoder outputs.
- """
-
- def __init__(self, dim):
- super(ScaledDotProductAttention, self).__init__()
- self.sqrt_dim = np.sqrt(dim)
-
- def forward(self, query, key, value):
- # Single-element attention is an identity operation. Skip backend kernel
- # dispatch for this degenerate case (for example, Triton on CUDA).
- if query.size(1) == 1 and key.size(1) == 1 and value.size(1) == 1:
- return value
-
- score = torch.bmm(query, key.transpose(1, 2)) / self.sqrt_dim
- attn = F.softmax(score, -1)
- context = torch.bmm(attn, value)
- return context
-
-
-class AttFusion(nn.Module):
- def __init__(self, feature_dim):
- super(AttFusion, self).__init__()
- self.att = ScaledDotProductAttention(feature_dim)
-
- def forward(self, x, record_len):
- split_x = self.regroup(x, record_len)
- C, W, H = split_x[0].shape[1:]
- out = []
- for xx in split_x:
- cav_num = xx.shape[0]
- xx = xx.view(cav_num, C, -1).permute(2, 0, 1)
- h = self.att(xx, xx, xx)
- h = h.permute(1, 2, 0).view(cav_num, C, W, H)[0, ...]
- out.append(h)
- return torch.stack(out)
-
- def regroup(self, x, record_len):
- cum_sum_len = torch.cumsum(record_len, dim=0)
- split_x = torch.tensor_split(x, cum_sum_len[:-1].cpu())
- return split_x
diff --git a/OpenCOOD/opencood/models/fuse_modules/swap_fusion_modules.py b/OpenCOOD/opencood/models/fuse_modules/swap_fusion_modules.py
deleted file mode 100644
index d14e7f6a..00000000
--- a/OpenCOOD/opencood/models/fuse_modules/swap_fusion_modules.py
+++ /dev/null
@@ -1,234 +0,0 @@
-"""
-This class is about swap fusion applications (also known as Fused Axial Attention)
-"""
-
-import torch
-from einops import rearrange
-from torch import nn, einsum
-from einops.layers.torch import Rearrange, Reduce
-
-from opencood.models.sub_modules.base_transformer import FeedForward, PreNormResidual
-
-
-# swap attention -> max_vit
-class Attention(nn.Module):
- """
- Unit Attention class. Todo: mask is not added yet.
-
- Parameters
- ----------
- dim: int
- Input feature dimension.
- dim_head: int
- The head dimension.
- dropout: float
- Dropout rate
- agent_size: int
- The agent can be different views, timestamps or vehicles.
- """
-
- def __init__(self, dim, dim_head=32, dropout=0.0, agent_size=6, window_size=7):
- super().__init__()
- assert (dim % dim_head) == 0, "dimension should be divisible by dimension per head"
-
- self.heads = dim // dim_head
- self.scale = dim_head**-0.5
- self.window_size = [agent_size, window_size, window_size]
-
- self.to_qkv = nn.Linear(dim, dim * 3, bias=False)
- self.attend = nn.Sequential(nn.Softmax(dim=-1))
-
- self.to_out = nn.Sequential(nn.Linear(dim, dim, bias=False), nn.Dropout(dropout))
-
- self.relative_position_bias_table = nn.Embedding(
- (2 * self.window_size[0] - 1) * (2 * self.window_size[1] - 1) * (2 * self.window_size[2] - 1), self.heads
- ) # 2*Wd-1 * 2*Wh-1 * 2*Ww-1, nH
-
- # get pair-wise relative position index for
- # each token inside the window
- coords_d = torch.arange(self.window_size[0])
- coords_h = torch.arange(self.window_size[1])
- coords_w = torch.arange(self.window_size[2])
- # 3, Wd, Wh, Ww
- coords = torch.stack(torch.meshgrid(coords_d, coords_h, coords_w))
- coords_flatten = torch.flatten(coords, 1) # 3, Wd*Wh*Ww
-
- # 3, Wd*Wh*Ww, Wd*Wh*Ww
- relative_coords = coords_flatten[:, :, None] - coords_flatten[:, None, :]
- # Wd*Wh*Ww, Wd*Wh*Ww, 3
- relative_coords = relative_coords.permute(1, 2, 0).contiguous()
- # shift to start from 0
- relative_coords[:, :, 0] += self.window_size[0] - 1
- relative_coords[:, :, 1] += self.window_size[1] - 1
- relative_coords[:, :, 2] += self.window_size[2] - 1
-
- relative_coords[:, :, 0] *= (2 * self.window_size[1] - 1) * (2 * self.window_size[2] - 1)
- relative_coords[:, :, 1] *= 2 * self.window_size[2] - 1
- relative_position_index = relative_coords.sum(-1) # Wd*Wh*Ww, Wd*Wh*Ww
- self.register_buffer("relative_position_index", relative_position_index)
-
- def forward(self, x, mask=None):
- # x shape: b, l, h, w, w_h, w_w, c
- batch, agent_size, height, width, window_height, window_width, _, _, h = *x.shape, x.device, self.heads # eighth variable is device
-
- # flatten
- x = rearrange(x, "b l x y w1 w2 d -> (b x y) (l w1 w2) d")
- # project for queries, keys, values
- q, k, v = self.to_qkv(x).chunk(3, dim=-1)
- # split heads
- q, k, v = map(lambda t: rearrange(t, "b n (h d) -> b h n d", h=h), (q, k, v))
- # scale
- q = q * self.scale
-
- # sim
- sim = einsum("b h i d, b h j d -> b h i j", q, k)
-
- # add positional bias
- bias = self.relative_position_bias_table(self.relative_position_index)
- sim = sim + rearrange(bias, "i j h -> h i j")
-
- # mask shape if exist: b x y w1 w2 e l
- if mask is not None:
- # b x y w1 w2 e l -> (b x y) 1 (l w1 w2)
- mask = rearrange(mask, "b x y w1 w2 e l -> (b x y) e (l w1 w2)")
- # (b x y) 1 1 (l w1 w2) = b h 1 n
- mask = mask.unsqueeze(1)
- sim = sim.masked_fill(mask == 0, -float("inf"))
-
- # attention
- attn = self.attend(sim)
- # aggregate
- out = einsum("b h i j, b h j d -> b h i d", attn, v)
- # merge heads
- out = rearrange(out, "b h (l w1 w2) d -> b l w1 w2 (h d)", l=agent_size, w1=window_height, w2=window_width)
-
- # combine heads out
- out = self.to_out(out)
- return rearrange(out, "(b x y) l w1 w2 d -> b l x y w1 w2 d", b=batch, x=height, y=width)
-
-
-class SwapFusionBlockMask(nn.Module):
- """
- Swap Fusion Block contains window attention and grid attention with
- mask enabled for multi-vehicle cooperation.
- """
-
- def __init__(self, input_dim, mlp_dim, dim_head, window_size, agent_size, drop_out):
- super(SwapFusionBlockMask, self).__init__()
-
- self.window_size = window_size
-
- self.window_attention = PreNormResidual(input_dim, Attention(input_dim, dim_head, drop_out, agent_size, window_size))
- self.window_ffd = PreNormResidual(input_dim, FeedForward(input_dim, mlp_dim, drop_out))
- self.grid_attention = PreNormResidual(input_dim, Attention(input_dim, dim_head, drop_out, agent_size, window_size))
- self.grid_ffd = PreNormResidual(input_dim, FeedForward(input_dim, mlp_dim, drop_out))
-
- def forward(self, x, mask):
- # x: b l c h w
- # mask: b h w 1 l
- # window attention -> grid attention
- mask_swap = mask
-
- # mask b h w 1 l -> b x y w1 w2 1 L
- mask_swap = rearrange(mask_swap, "b (x w1) (y w2) e l -> b x y w1 w2 e l", w1=self.window_size, w2=self.window_size)
- x = rearrange(x, "b m d (x w1) (y w2) -> b m x y w1 w2 d", w1=self.window_size, w2=self.window_size)
- x = self.window_attention(x, mask=mask_swap)
- x = self.window_ffd(x)
- x = rearrange(x, "b m x y w1 w2 d -> b m d (x w1) (y w2)")
-
- # grid attention
- mask_swap = mask
- mask_swap = rearrange(mask_swap, "b (w1 x) (w2 y) e l -> b x y w1 w2 e l", w1=self.window_size, w2=self.window_size)
- x = rearrange(x, "b m d (w1 x) (w2 y) -> b m x y w1 w2 d", w1=self.window_size, w2=self.window_size)
- x = self.grid_attention(x, mask=mask_swap)
- x = self.grid_ffd(x)
- x = rearrange(x, "b m x y w1 w2 d -> b m d (w1 x) (w2 y)")
-
- return x
-
-
-class SwapFusionBlock(nn.Module):
- """
- Swap Fusion Block contains window attention and grid attention.
- """
-
- def __init__(self, input_dim, mlp_dim, dim_head, window_size, agent_size, drop_out):
- super(SwapFusionBlock, self).__init__()
- # b = batch * max_cav
- self.block = nn.Sequential(
- Rearrange("b m d (x w1) (y w2) -> b m x y w1 w2 d", w1=window_size, w2=window_size),
- PreNormResidual(input_dim, Attention(input_dim, dim_head, drop_out, agent_size, window_size)),
- PreNormResidual(input_dim, FeedForward(input_dim, mlp_dim, drop_out)),
- Rearrange("b m x y w1 w2 d -> b m d (x w1) (y w2)"),
- Rearrange("b m d (w1 x) (w2 y) -> b m x y w1 w2 d", w1=window_size, w2=window_size),
- PreNormResidual(input_dim, Attention(input_dim, dim_head, drop_out, agent_size, window_size)),
- PreNormResidual(input_dim, FeedForward(input_dim, mlp_dim, drop_out)),
- Rearrange("b m x y w1 w2 d -> b m d (w1 x) (w2 y)"),
- )
-
- def forward(self, x, mask=None):
- # todo: add mask operation later for mulit-agents
- x = self.block(x)
- return x
-
-
-class SwapFusionEncoder(nn.Module):
- """
- Data rearrange -> swap block -> mlp_head
- """
-
- def __init__(self, args):
- super(SwapFusionEncoder, self).__init__()
-
- self.layers = nn.ModuleList([])
- self.depth = args["depth"]
-
- # block related
- input_dim = args["input_dim"]
- mlp_dim = args["mlp_dim"]
- agent_size = args["agent_size"]
- window_size = args["window_size"]
- drop_out = args["drop_out"]
- dim_head = args["dim_head"]
-
- self.mask = False
- if "mask" in args:
- self.mask = args["mask"]
-
- for i in range(self.depth):
- if self.mask:
- block = SwapFusionBlockMask(input_dim, mlp_dim, dim_head, window_size, agent_size, drop_out)
-
- else:
- block = SwapFusionBlock(input_dim, mlp_dim, dim_head, window_size, agent_size, drop_out)
- self.layers.append(block)
-
- # mlp head
- self.mlp_head = nn.Sequential(
- Reduce("b m d h w -> b d h w", "mean"),
- Rearrange("b d h w -> b h w d"),
- nn.LayerNorm(input_dim),
- nn.Linear(input_dim, input_dim),
- Rearrange("b h w d -> b d h w"),
- )
-
- def forward(self, x, mask=None):
- for stage in self.layers:
- x = stage(x, mask=mask)
- return self.mlp_head(x)
-
-
-if __name__ == "__main__":
- import os
-
- os.environ["CUDA_VISIBLE_DEVICES"] = "1"
- args = {"input_dim": 512, "mlp_dim": 512, "agent_size": 4, "window_size": 8, "dim_head": 4, "drop_out": 0.1, "depth": 2, "mask": True}
- block = SwapFusionEncoder(args)
- block.cuda()
- test_data = torch.rand(1, 4, 512, 32, 32)
- test_data = test_data.cuda()
- mask = torch.ones(1, 32, 32, 1, 4)
- mask = mask.cuda()
-
- output = block(test_data, mask)
- print(output)
diff --git a/OpenCOOD/opencood/models/fuse_modules/v2v_fuse.py b/OpenCOOD/opencood/models/fuse_modules/v2v_fuse.py
deleted file mode 100644
index 756ca830..00000000
--- a/OpenCOOD/opencood/models/fuse_modules/v2v_fuse.py
+++ /dev/null
@@ -1,135 +0,0 @@
-"""
-Implementation of V2VNet Fusion
-"""
-
-import torch
-import torch.nn as nn
-
-from opencood.models.sub_modules.torch_transformation_utils import (
- get_discretized_transformation_matrix,
- get_transformation_matrix,
- warp_affine,
- get_rotated_roi,
-)
-from opencood.models.sub_modules.convgru import ConvGRU
-
-
-class V2VNetFusion(nn.Module):
- def __init__(self, args):
- super(V2VNetFusion, self).__init__()
-
- in_channels = args["in_channels"]
- H, W = args["conv_gru"]["H"], args["conv_gru"]["W"]
- kernel_size = args["conv_gru"]["kernel_size"]
- num_gru_layers = args["conv_gru"]["num_layers"]
-
- self.discrete_ratio = args["voxel_size"][0]
- self.downsample_rate = args["downsample_rate"]
- self.num_iteration = args["num_iteration"]
- self.gru_flag = args["gru_flag"]
- self.agg_operator = args["agg_operator"]
-
- self.msg_cnn = nn.Conv2d(in_channels * 2, in_channels, kernel_size=3, stride=1, padding=1)
- self.conv_gru = ConvGRU(
- input_size=(H, W),
- input_dim=in_channels * 2,
- hidden_dim=[in_channels],
- kernel_size=kernel_size,
- num_layers=num_gru_layers,
- batch_first=True,
- bias=True,
- return_all_layers=False,
- )
- self.mlp = nn.Linear(in_channels, in_channels)
-
- def regroup(self, x, record_len):
- cum_sum_len = torch.cumsum(record_len, dim=0)
- split_x = torch.tensor_split(x, cum_sum_len[:-1].cpu())
- return split_x
-
- def forward(self, x, record_len, pairwise_t_matrix):
- """
- Fusion forwarding.
-
- Parameters
- ----------
- x : torch.Tensor
- input data, (B, C, H, W)
-
- record_len : list
- shape: (B)
-
- pairwise_t_matrix : torch.Tensor
- The transformation matrix from each cav to ego,
- shape: (B, L, L, 4, 4)
-
- Returns
- -------
- Fused feature.
- """
- _, C, H, W = x.shape
- B, L = pairwise_t_matrix.shape[:2]
-
- # split x:[(L1, C, H, W), (L2, C, H, W)]
- split_x = self.regroup(x, record_len)
- # (B,L,L,2,3)
- pairwise_t_matrix = get_discretized_transformation_matrix(
- pairwise_t_matrix.reshape(-1, L, 4, 4), self.discrete_ratio, self.downsample_rate
- ).reshape(B, L, L, 2, 3)
- # (B*L,L,1,H,W)
- roi_mask = get_rotated_roi((B * L, L, 1, H, W), pairwise_t_matrix.reshape(B * L * L, 2, 3))
- roi_mask = roi_mask.reshape(B, L, L, 1, H, W)
- batch_node_features = split_x
-
- # iteratively update the features for num_iteration times
- for _ in range(self.num_iteration):
- batch_updated_node_features = []
- # iterate each batch
- for b in range(B):
- # number of valid agent
- N = record_len[b]
- # (N,N,4,4)
- # t_matrix[i, j]-> from i to j
- t_matrix = pairwise_t_matrix[b][:N, :N, :, :]
- updated_node_features = []
- # update each node i
- for i in range(N):
- # (N,1,H,W)
- mask = roi_mask[b, :N, i, ...]
-
- current_t_matrix = t_matrix[:, i, :, :]
- current_t_matrix = get_transformation_matrix(current_t_matrix, (H, W))
-
- # (N,C,H,W)
- neighbor_feature = warp_affine(batch_node_features[b], current_t_matrix, (H, W))
- # (N,C,H,W)
- ego_agent_feature = batch_node_features[b][i].unsqueeze(0).repeat(N, 1, 1, 1)
- # (N,2C,H,W)
- neighbor_feature = torch.cat([neighbor_feature, ego_agent_feature], dim=1)
- # (N,C,H,W)
- message = self.msg_cnn(neighbor_feature) * mask
-
- # (C,H,W)
- if self.agg_operator == "avg":
- agg_feature = torch.mean(message, dim=0)
- elif self.agg_operator == "max":
- agg_feature = torch.max(message, dim=0)[0]
- else:
- raise ValueError("agg_operator has wrong value")
- # (2C, H, W)
- cat_feature = torch.cat([batch_node_features[b][i, ...], agg_feature], dim=0)
- # (C,H,W)
- if self.gru_flag:
- gru_out = self.conv_gru(cat_feature.unsqueeze(0).unsqueeze(0))[0][0].squeeze(0).squeeze(0)
- else:
- gru_out = batch_node_features[b][i, ...] + agg_feature
- updated_node_features.append(gru_out.unsqueeze(0))
- # (N,C,H,W)
- batch_updated_node_features.append(torch.cat(updated_node_features, dim=0))
- batch_node_features = batch_updated_node_features
- # (B,C,H,W)
- out = torch.cat([itm[0, ...].unsqueeze(0) for itm in batch_node_features], dim=0)
- # (B,C,H,W)
- out = self.mlp(out.permute(0, 2, 3, 1)).permute(0, 3, 1, 2)
-
- return out
diff --git a/OpenCOOD/opencood/models/fuse_modules/v2xvit_basic.py b/OpenCOOD/opencood/models/fuse_modules/v2xvit_basic.py
deleted file mode 100644
index 6874ee2c..00000000
--- a/OpenCOOD/opencood/models/fuse_modules/v2xvit_basic.py
+++ /dev/null
@@ -1,190 +0,0 @@
-import math
-
-import torch
-from torch import nn
-
-from opencood.models.fuse_modules.hmsa import HGTCavAttention
-from opencood.models.fuse_modules.mswin import PyramidWindowAttention
-from opencood.models.sub_modules.base_transformer import CavAttention, FeedForward, PreNorm
-from opencood.models.sub_modules.torch_transformation_utils import (
- get_discretized_transformation_matrix,
- get_roi_and_cav_mask,
- get_transformation_matrix,
- warp_affine,
-)
-
-
-class STTF(nn.Module):
- def __init__(self, args):
- super(STTF, self).__init__()
- self.discrete_ratio = args["voxel_size"][0]
- self.downsample_rate = args["downsample_rate"]
-
- def forward(self, x, mask, spatial_correction_matrix):
- x = x.permute(0, 1, 4, 2, 3)
- dist_correction_matrix = get_discretized_transformation_matrix(spatial_correction_matrix, self.discrete_ratio, self.downsample_rate)
- # Only compensate non-ego vehicles
- B, L, C, H, W = x.shape
-
- T = get_transformation_matrix(dist_correction_matrix[:, 1:, :, :].reshape(-1, 2, 3), (H, W))
- cav_features = warp_affine(x[:, 1:, :, :, :].reshape(-1, C, H, W), T, (H, W))
- cav_features = cav_features.reshape(B, -1, C, H, W)
- x = torch.cat([x[:, 0, :, :, :].unsqueeze(1), cav_features], dim=1)
- x = x.permute(0, 1, 3, 4, 2)
- return x
-
-
-class RelTemporalEncoding(nn.Module):
- """
- Implement the Temporal Encoding (Sinusoid) function.
- """
-
- def __init__(self, n_hid, RTE_ratio, max_len=100, dropout=0.2):
- super(RelTemporalEncoding, self).__init__()
- position = torch.arange(0.0, max_len).unsqueeze(1)
- div_term = torch.exp(torch.arange(0, n_hid, 2) * -(math.log(10000.0) / n_hid))
- emb = nn.Embedding(max_len, n_hid)
- emb.weight.data[:, 0::2] = torch.sin(position * div_term) / math.sqrt(n_hid)
- emb.weight.data[:, 1::2] = torch.cos(position * div_term) / math.sqrt(n_hid)
- emb.requires_grad = False
- self.RTE_ratio = RTE_ratio
- self.emb = emb
- self.lin = nn.Linear(n_hid, n_hid)
-
- def forward(self, x, t):
- # When t has unit of 50ms, rte_ratio=1.
- # So we can train on 100ms but test on 50ms
- return x + self.lin(self.emb(t * self.RTE_ratio)).unsqueeze(0).unsqueeze(1)
-
-
-class RTE(nn.Module):
- def __init__(self, dim, RTE_ratio=2):
- super(RTE, self).__init__()
- self.RTE_ratio = RTE_ratio
-
- self.emb = RelTemporalEncoding(dim, RTE_ratio=self.RTE_ratio)
-
- def forward(self, x, dts):
- # x: (B,L,H,W,C)
- # dts: (B,L)
- rte_batch = []
- for b in range(x.shape[0]):
- rte_list = []
- for i in range(x.shape[1]):
- rte_list.append(self.emb(x[b, i, :, :, :], dts[b, i]).unsqueeze(0))
- rte_batch.append(torch.cat(rte_list, dim=0).unsqueeze(0))
- return torch.cat(rte_batch, dim=0)
-
-
-class V2XFusionBlock(nn.Module):
- def __init__(self, num_blocks, cav_att_config, pwindow_config):
- super().__init__()
- # first multi-agent attention and then multi-window attention
- self.layers = nn.ModuleList([])
- self.num_blocks = num_blocks
-
- for _ in range(num_blocks):
- att = (
- HGTCavAttention(
- cav_att_config["dim"], heads=cav_att_config["heads"], dim_head=cav_att_config["dim_head"], dropout=cav_att_config["dropout"]
- )
- if cav_att_config["use_hetero"]
- else CavAttention(
- cav_att_config["dim"], heads=cav_att_config["heads"], dim_head=cav_att_config["dim_head"], dropout=cav_att_config["dropout"]
- )
- )
- self.layers.append(
- nn.ModuleList(
- [
- PreNorm(cav_att_config["dim"], att),
- PreNorm(
- cav_att_config["dim"],
- PyramidWindowAttention(
- pwindow_config["dim"],
- heads=pwindow_config["heads"],
- dim_heads=pwindow_config["dim_head"],
- drop_out=pwindow_config["dropout"],
- window_size=pwindow_config["window_size"],
- relative_pos_embedding=pwindow_config["relative_pos_embedding"],
- fuse_method=pwindow_config["fusion_method"],
- ),
- ),
- ]
- )
- )
-
- def forward(self, x, mask, prior_encoding):
- for cav_attn, pwindow_attn in self.layers:
- x = cav_attn(x, mask=mask, prior_encoding=prior_encoding) + x
- x = pwindow_attn(x) + x
- return x
-
-
-class V2XTEncoder(nn.Module):
- def __init__(self, args):
- super().__init__()
-
- cav_att_config = args["cav_att_config"]
- pwindow_att_config = args["pwindow_att_config"]
- feed_config = args["feed_forward"]
-
- num_blocks = args["num_blocks"]
- depth = args["depth"]
- mlp_dim = feed_config["mlp_dim"]
- dropout = feed_config["dropout"]
-
- self.downsample_rate = args["sttf"]["downsample_rate"]
- self.discrete_ratio = args["sttf"]["voxel_size"][0]
- self.use_roi_mask = args["use_roi_mask"]
- self.use_RTE = cav_att_config["use_RTE"]
- self.RTE_ratio = cav_att_config["RTE_ratio"]
- self.sttf = STTF(args["sttf"])
- # adjust the channel numbers from 256+3 -> 256
- self.prior_feed = nn.Linear(cav_att_config["dim"] + 3, cav_att_config["dim"]) # noqa: DC05
- self.layers = nn.ModuleList([])
- if self.use_RTE:
- self.rte = RTE(cav_att_config["dim"], self.RTE_ratio)
- for _ in range(depth):
- self.layers.append(
- nn.ModuleList(
- [
- V2XFusionBlock(num_blocks, cav_att_config, pwindow_att_config),
- PreNorm(cav_att_config["dim"], FeedForward(cav_att_config["dim"], mlp_dim, dropout=dropout)),
- ]
- )
- )
-
- def forward(self, x, mask, spatial_correction_matrix):
- # transform the features to the current timestamp
- # velocity, time_delay, infra
- # (B,L,H,W,3)
- prior_encoding = x[..., -3:]
- # (B,L,H,W,C)
- x = x[..., :-3]
- if self.use_RTE:
- # dt: (B,L)
- dt = prior_encoding[:, :, 0, 0, 1].to(torch.int)
- x = self.rte(x, dt)
- x = self.sttf(x, mask, spatial_correction_matrix)
- com_mask = (
- mask.unsqueeze(1).unsqueeze(2).unsqueeze(3)
- if not self.use_roi_mask
- else get_roi_and_cav_mask(x.shape, mask, spatial_correction_matrix, self.discrete_ratio, self.downsample_rate)
- )
- for attn, ff in self.layers:
- x = attn(x, mask=com_mask, prior_encoding=prior_encoding)
- x = ff(x) + x
- return x
-
-
-class V2XTransformer(nn.Module):
- def __init__(self, args):
- super(V2XTransformer, self).__init__()
-
- encoder_args = args["encoder"]
- self.encoder = V2XTEncoder(encoder_args)
-
- def forward(self, x, mask, spatial_correction_matrix):
- output = self.encoder(x, mask, spatial_correction_matrix)
- output = output[:, 0]
- return output
diff --git a/OpenCOOD/opencood/models/fuse_modules/where2comm_fuse.py b/OpenCOOD/opencood/models/fuse_modules/where2comm_fuse.py
deleted file mode 100644
index 6d4b3665..00000000
--- a/OpenCOOD/opencood/models/fuse_modules/where2comm_fuse.py
+++ /dev/null
@@ -1,237 +0,0 @@
-"""
-Implementation of Where2comm fusion.
-"""
-
-import numpy as np
-import random
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-
-from opencood.models.fuse_modules.self_attn import ScaledDotProductAttention
-
-
-class Communication(nn.Module):
- def __init__(self, args):
- super(Communication, self).__init__()
- # Threshold of objectiveness
- self.threshold = args["threshold"]
- if "gaussian_smooth" in args:
- # Gaussian Smooth
- self.smooth = True
- kernel_size = args["gaussian_smooth"]["k_size"]
- c_sigma = args["gaussian_smooth"]["c_sigma"]
- self.gaussian_filter = nn.Conv2d(1, 1, kernel_size=kernel_size, stride=1, padding=(kernel_size - 1) // 2)
- self.init_gaussian_filter(kernel_size, c_sigma)
- self.gaussian_filter.requires_grad = False
- else:
- self.smooth = False
-
- def init_gaussian_filter(self, k_size=5, sigma=1.0):
- center = k_size // 2
- x, y = np.mgrid[0 - center : k_size - center, 0 - center : k_size - center]
- gaussian_kernel = 1 / (2 * np.pi * sigma) * np.exp(-(np.square(x) + np.square(y)) / (2 * np.square(sigma)))
-
- self.gaussian_filter.weight.data = torch.Tensor(gaussian_kernel).to(self.gaussian_filter.weight.device).unsqueeze(0).unsqueeze(0)
- self.gaussian_filter.bias.data.zero_()
-
- def build_mask(self, confidence_map, *, force_first: bool):
- """
- Build the spatial transmission mask for one group of agents.
-
- Parameters
- ----------
- confidence_map : torch.Tensor
- Per-agent classification logits.
- force_first : bool
- Whether the first feature belongs to the local receiver and must
- remain unmasked.
-
- Returns
- -------
- tuple[torch.Tensor, torch.Tensor]
- Binary communication mask and selected-location ratio.
- """
- _, _, height, width = confidence_map.shape
- original_map, _ = confidence_map.sigmoid().max(dim=1, keepdim=True)
- communication_map = self.gaussian_filter(original_map) if self.smooth else original_map
- agent_count = communication_map.shape[0]
-
- if self.training:
- selected_count = int(height * width * random.uniform(0, 1))
- flattened_map = communication_map.reshape(agent_count, height * width)
- _, indices = torch.topk(flattened_map, k=selected_count, sorted=False)
- communication_mask = torch.zeros_like(flattened_map)
- selected_values = torch.ones(
- agent_count,
- selected_count,
- dtype=communication_map.dtype,
- device=communication_map.device,
- )
- communication_mask = torch.scatter(
- communication_mask,
- -1,
- indices,
- selected_values,
- ).reshape(agent_count, 1, height, width)
- elif self.threshold:
- communication_mask = torch.where(
- communication_map > self.threshold,
- torch.ones_like(communication_map),
- torch.zeros_like(communication_map),
- )
- else:
- communication_mask = torch.ones_like(communication_map)
-
- communication_rate = communication_mask.sum() / (agent_count * height * width)
- if force_first:
- communication_mask[0] = 1
- return communication_mask, communication_rate
-
- def forward(self, batch_confidence_maps, B):
- """
- Args:
- batch_confidence_maps: [(L1, H, W), (L2, H, W), ...]
- """
-
- communication_masks = []
- communication_rates = []
- for b in range(B):
- communication_mask, communication_rate = self.build_mask(
- batch_confidence_maps[b],
- force_first=True,
- )
- communication_masks.append(communication_mask)
- communication_rates.append(communication_rate)
- communication_rates = sum(communication_rates) / B
- communication_masks = torch.cat(communication_masks, dim=0)
- return communication_masks, communication_rates
-
-
-class AttentionFusion(nn.Module):
- def __init__(self, feature_dim):
- super(AttentionFusion, self).__init__()
- self.att = ScaledDotProductAttention(feature_dim)
-
- def forward(self, x):
- cav_num, C, H, W = x.shape
- x = x.view(cav_num, C, -1).permute(2, 0, 1) # (H*W, cav_num, C), perform self attention on each pixel
- x = self.att(x, x, x)
- x = x.permute(1, 2, 0).view(cav_num, C, H, W)[0] # C, W, H before
- return x
-
-
-class Where2comm(nn.Module):
- def __init__(self, args):
- super(Where2comm, self).__init__()
- self.discrete_ratio = args["voxel_size"][0]
- self.downsample_rate = args["downsample_rate"]
-
- self.fully = args["fully"]
- if self.fully:
- print("constructing a fully connected communication graph")
- else:
- print("constructing a partially connected communication graph")
-
- self.multi_scale = args["multi_scale"]
- if self.multi_scale:
- layer_nums = args["layer_nums"]
- num_filters = args["num_filters"]
- self.num_levels = len(layer_nums)
- self.fuse_modules = nn.ModuleList()
- for idx in range(self.num_levels):
- fuse_network = AttentionFusion(num_filters[idx])
- self.fuse_modules.append(fuse_network)
- else:
- self.fuse_modules = AttentionFusion(args["in_channels"])
-
- self.naive_communication = Communication(args["communication"])
-
- def regroup(self, x, record_len):
- cum_sum_len = torch.cumsum(record_len, dim=0)
- split_x = torch.tensor_split(x, cum_sum_len[:-1].cpu())
- return split_x
-
- def forward(self, x, psm_single, record_len, pairwise_t_matrix, backbone=None):
- """
- Fusion forwarding.
-
- Parameters:
- x: Input data, (sum(n_cav), C, H, W).
- record_len: List, (B).
- pairwise_t_matrix: The transformation matrix from each cav to ego, (B, L, L, 4, 4).
-
- Returns:
- Fused feature.
- """
-
- _, C, H, W = x.shape
- B = pairwise_t_matrix.shape[0]
-
- if self.multi_scale:
- ups = []
-
- for i in range(self.num_levels):
- x = backbone.blocks[i](x)
-
- # 1. Communication (mask the features)
- if i == 0:
- if self.fully:
- communication_rates = torch.tensor(1).to(x.device)
- else:
- # Prune
- batch_confidence_maps = self.regroup(psm_single, record_len)
- communication_masks, communication_rates = self.naive_communication(batch_confidence_maps, B)
- if x.shape[-1] != communication_masks.shape[-1]:
- communication_masks = F.interpolate(
- communication_masks, size=(x.shape[-2], x.shape[-1]), mode="bilinear", align_corners=False
- )
- x = x * communication_masks
-
- # 2. Split the features
- # split_x: [(L1, C, H, W), (L2, C, H, W), ...]
- # For example [[2, 256, 48, 176], [1, 256, 48, 176], ...]
- batch_node_features = self.regroup(x, record_len)
-
- # 3. Fusion
- x_fuse = []
- for b in range(B):
- neighbor_feature = batch_node_features[b]
- x_fuse.append(self.fuse_modules[i](neighbor_feature))
- x_fuse = torch.stack(x_fuse)
-
- # 4. Deconv
- if len(backbone.deblocks) > 0:
- ups.append(backbone.deblocks[i](x_fuse))
- else:
- ups.append(x_fuse)
-
- if len(ups) > 1:
- x_fuse = torch.cat(ups, dim=1)
- elif len(ups) == 1:
- x_fuse = ups[0]
-
- if len(backbone.deblocks) > self.num_levels:
- x_fuse = backbone.deblocks[-1](x_fuse)
- else:
- # 1. Communication (mask the features)
- if self.fully:
- communication_rates = torch.tensor(1).to(x.device)
- else:
- # Prune
- batch_confidence_maps = self.regroup(psm_single, record_len)
- communication_masks, communication_rates = self.naive_communication(batch_confidence_maps, B)
- x = x * communication_masks
-
- # 2. Split the features
- # split_x: [(L1, C, H, W), (L2, C, H, W), ...]
- # For example [[2, 256, 48, 176], [1, 256, 48, 176], ...]
- batch_node_features = self.regroup(x, record_len)
-
- # 3. Fusion
- x_fuse = []
- for b in range(B):
- neighbor_feature = batch_node_features[b]
- x_fuse.append(self.fuse_modules(neighbor_feature))
- x_fuse = torch.stack(x_fuse)
- return x_fuse, communication_rates
diff --git a/OpenCOOD/opencood/models/pixor.py b/OpenCOOD/opencood/models/pixor.py
deleted file mode 100644
index 4f54753b..00000000
--- a/OpenCOOD/opencood/models/pixor.py
+++ /dev/null
@@ -1,266 +0,0 @@
-import math
-
-import torch.nn as nn
-
-
-def conv3x3(in_planes, out_planes, stride=1, bias=False):
- """3x3 convolution with padding"""
- return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, padding=1, bias=bias)
-
-
-class BasicBlock(nn.Module):
- expansion = 1
-
- def __init__(self, in_planes, planes, stride=1, downsample=None):
- super(BasicBlock, self).__init__()
- self.conv1 = conv3x3(in_planes, planes, stride, bias=True)
- self.bn1 = nn.BatchNorm2d(planes)
- self.relu = nn.ReLU(inplace=True)
- self.conv2 = conv3x3(planes, planes, bias=True)
- self.bn2 = nn.BatchNorm2d(planes)
- self.downsample = downsample
- self.stride = stride
-
- def forward(self, x):
- residual = x
-
- out = self.conv1(x)
- # out = self.bn1(out)
- out = self.relu(out)
-
- out = self.conv2(out)
- # out = self.bn2(out)
-
- if self.downsample is not None:
- residual = self.downsample(x)
-
- out += residual
- out = self.relu(out)
-
- return out
-
-
-class Bottleneck(nn.Module):
- expansion = 4
-
- def __init__(self, in_planes, planes, stride=1, downsample=None, use_bn=True):
- super(Bottleneck, self).__init__()
- bias = not use_bn
- self.use_bn = use_bn
- self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=1, bias=bias)
- self.bn1 = nn.BatchNorm2d(planes)
- self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, padding=1, bias=bias)
- self.bn2 = nn.BatchNorm2d(planes)
- self.conv3 = nn.Conv2d(planes, self.expansion * planes, kernel_size=1, bias=bias)
- self.bn3 = nn.BatchNorm2d(self.expansion * planes)
- self.downsample = downsample
- self.relu = nn.ReLU(inplace=True)
-
- def forward(self, x):
- """
- Forward pass of residual block.
- Parameters
- ----------
- x : torch.Tensor
- Shape (N, C, W, L).
-
- Returns
- -------
- out : torch.Tensor
- Shape (N, self.expansion*planes, W/stride, L/stride).
- """
- residual = x
- # (N, planes, W, L)
- out = self.conv1(x)
- if self.use_bn:
- out = self.bn1(out)
- out = self.relu(out)
- # (N, planes, W/stride, L/stride)
- out = self.conv2(out)
- if self.use_bn:
- out = self.bn2(out)
- out = self.relu(out)
- # (N, self.expansion*planes, W/stride, L/stride)
- out = self.conv3(out)
- if self.use_bn:
- out = self.bn3(out)
-
- if self.downsample is not None:
- # (N, self.expansion*planes, W/2, L/2)
- residual = self.downsample(x)
- out = self.relu(residual + out)
- return out
-
-
-class BackBone(nn.Module):
- def __init__(self, block, num_block, geom, use_bn=True):
- super(BackBone, self).__init__()
-
- self.use_bn = use_bn
-
- # Block 1
- self.conv1 = conv3x3(geom["input_shape"][-1], 32)
- self.conv2 = conv3x3(32, 32)
- self.bn1 = nn.BatchNorm2d(32)
- self.bn2 = nn.BatchNorm2d(32)
- self.relu = nn.ReLU(inplace=True)
-
- # Block 2-5
- self.in_planes = 32
- self.block2 = self._make_layer(block, 24, num_blocks=num_block[0])
- self.block3 = self._make_layer(block, 48, num_blocks=num_block[1])
- self.block4 = self._make_layer(block, 64, num_blocks=num_block[2])
- self.block5 = self._make_layer(block, 96, num_blocks=num_block[3])
-
- # Lateral layers
- self.latlayer1 = nn.Conv2d(384, 196, kernel_size=1, stride=1, padding=0)
- self.latlayer2 = nn.Conv2d(256, 128, kernel_size=1, stride=1, padding=0)
- self.latlayer3 = nn.Conv2d(192, 96, kernel_size=1, stride=1, padding=0)
-
- # Top-down layers
- self.deconv1 = nn.ConvTranspose2d(196, 128, kernel_size=3, stride=2, padding=1, output_padding=1)
- p = 0 if geom["label_shape"][1] == 175 else 1
- self.deconv2 = nn.ConvTranspose2d(128, 96, kernel_size=3, stride=2, padding=1, output_padding=(1, p))
-
- def encode(self, x):
- x = self.conv1(x)
- if self.use_bn:
- x = self.bn1(x)
- x = self.relu(x)
-
- x = self.conv2(x)
- if self.use_bn:
- x = self.bn2(x)
- c1 = self.relu(x)
-
- # bottom up layers
- c2 = self.block2(c1)
- c3 = self.block3(c2)
- c4 = self.block4(c3)
- c5 = self.block5(c4)
-
- return c3, c4, c5
-
- def decode(self, c3, c4, c5):
- l5 = self.latlayer1(c5)
- l4 = self.latlayer2(c4)
- p5 = l4 + self.deconv1(l5)
- l3 = self.latlayer3(c3)
- p4 = l3 + self.deconv2(p5)
-
- return p4
-
- def forward(self, x):
- c3, c4, c5 = self.encode(x)
- p4 = self.decode(c3, c4, c5)
-
- return p4
-
- def _make_layer(self, block, planes, num_blocks):
- if self.use_bn:
- # downsample the H*W by 1/2
- downsample = nn.Sequential(
- nn.Conv2d(self.in_planes, planes * block.expansion, kernel_size=1, stride=2, bias=False), nn.BatchNorm2d(planes * block.expansion)
- )
- else:
- downsample = nn.Conv2d(self.in_planes, planes * block.expansion, kernel_size=1, stride=2, bias=True)
-
- layers = [block(self.in_planes, planes, stride=2, downsample=downsample)]
-
- self.in_planes = planes * block.expansion
- for i in range(1, num_blocks):
- layers.append(block(self.in_planes, planes, stride=1))
- self.in_planes = planes * block.expansion
- return nn.Sequential(*layers)
-
-
-class Header(nn.Module):
- def __init__(self, use_bn=True):
- super(Header, self).__init__()
-
- self.use_bn = use_bn
- bias = not use_bn
- self.conv1 = conv3x3(96, 96, bias=bias)
- self.bn1 = nn.BatchNorm2d(96)
- self.conv2 = conv3x3(96, 96, bias=bias)
- self.bn2 = nn.BatchNorm2d(96)
- self.conv3 = conv3x3(96, 96, bias=bias)
- self.bn3 = nn.BatchNorm2d(96)
- self.conv4 = conv3x3(96, 96, bias=bias)
- self.bn4 = nn.BatchNorm2d(96)
-
- self.clshead = conv3x3(96, 1, bias=True)
- self.reghead = conv3x3(96, 6, bias=True)
-
- def forward(self, x):
- x = self.conv1(x)
- if self.use_bn:
- x = self.bn1(x)
- x = self.conv2(x)
- if self.use_bn:
- x = self.bn2(x)
- x = self.conv3(x)
- if self.use_bn:
- x = self.bn3(x)
- x = self.conv4(x)
- if self.use_bn:
- x = self.bn4(x)
-
- cls = self.clshead(x)
- reg = self.reghead(x)
-
- return cls, reg
-
-
-class PIXOR(nn.Module):
- """
- The Pixor backbone. The input of PIXOR nn module is a tensor of
- [batch_size, height, weight, channel], The output of PIXOR nn module
- is also a tensor of [batch_size, height/4, weight/4, channel]. Note that
- we convert the dimensions to [C, H, W] for PyTorch's nn.Conv2d functions
-
- Parameters
- ----------
- args : dict
- The arguments of the model.
-
- Attributes
- ----------
- backbone : opencood.object
- The backbone used to extract features.
- header : opencood.object
- Header used to predict the classification and coordinates.
- """
-
- def __init__(self, args):
- super(PIXOR, self).__init__()
- geom = args["geometry_param"]
- use_bn = args["use_bn"]
- self.backbone = BackBone(Bottleneck, [3, 6, 6, 3], geom, use_bn)
- self.header = Header(use_bn)
-
- for m in self.modules():
- if isinstance(m, nn.Conv2d):
- n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
- m.weight.data.normal_(0, math.sqrt(2.0 / n))
- elif isinstance(m, nn.BatchNorm2d):
- m.weight.data.fill_(1)
- m.bias.data.zero_()
-
- prior = 0.01
- self.header.clshead.weight.data.fill_(-math.log((1.0 - prior) / prior))
- self.header.clshead.bias.data.fill_(0)
- self.header.reghead.weight.data.fill_(0)
- self.header.reghead.bias.data.fill_(0)
-
- def forward(self, data_dict):
- bev_input = data_dict["processed_lidar"]["bev_input"]
-
- features = self.backbone(bev_input)
- # cls -- (N, 1, W/4, L/4)
- # reg -- (N, 6, W/4, L/4)
- cls, reg = self.header(features)
-
- output_dict = {"cls": cls, "reg": reg}
-
- return output_dict
diff --git a/OpenCOOD/opencood/models/pixor_intermediate.py b/OpenCOOD/opencood/models/pixor_intermediate.py
deleted file mode 100644
index ff880c59..00000000
--- a/OpenCOOD/opencood/models/pixor_intermediate.py
+++ /dev/null
@@ -1,139 +0,0 @@
-import math
-
-import torch.nn as nn
-
-from opencood.models.communication_adapters.intermediate import MultiScaleFeatureCommunicationAdapter
-from opencood.models.fuse_modules.self_attn import AttFusion
-from opencood.models.pixor import Bottleneck, BackBone, Header
-
-
-class BackBoneIntermediate(BackBone):
- def __init__(self, block, num_block, geom, use_bn=True):
- super(BackBoneIntermediate, self).__init__(block, num_block, geom, use_bn)
-
- self.fusion_net3 = AttFusion(192)
- self.fusion_net4 = AttFusion(256)
- self.fusion_net5 = AttFusion(384)
-
- def forward(self, x, record_len):
- return self.fuse_agents(self.encode(x), record_len)
-
- def fuse_agents(self, feature_maps, record_len):
- """
- Fuse encoded PIXOR scales and run the feature pyramid decoder.
-
- Parameters
- ----------
- feature_maps : tuple[torch.Tensor, torch.Tensor, torch.Tensor]
- Per-agent ``c3``, ``c4``, and ``c5`` feature maps.
- record_len : torch.Tensor
- Number of available agents in each sample.
-
- Returns
- -------
- torch.Tensor
- Decoded fused PIXOR feature map.
- """
- c3, c4, c5 = feature_maps
- c5 = self.fusion_net5(c5, record_len)
- c4 = self.fusion_net4(c4, record_len)
- c3 = self.fusion_net3(c3, record_len)
- return self.decode(c3, c4, c5)
-
-
-class PIXORIntermediate(nn.Module):
- """
- The Pixor backbone. The input of PIXOR nn module is a tensor of
- [batch_size, height, weight, channel], The output of PIXOR nn module
- is also a tensor of [batch_size, height/4, weight/4, channel]. Note that
- we convert the dimensions to [C, H, W] for PyTorch's nn.Conv2d functions
-
- Parameters
- ----------
- args : dict
- The arguments of the model.
-
- Attributes
- ----------
- backbone : opencood.object
- The backbone used to extract features.
- header : opencood.object
- Header used to predict the classification and coordinates.
- """
-
- communication_adapter_class = MultiScaleFeatureCommunicationAdapter
-
- def __init__(self, args):
- super(PIXORIntermediate, self).__init__()
- geom = args["geometry_param"]
- use_bn = args["use_bn"]
- self.backbone = BackBoneIntermediate(Bottleneck, [3, 6, 6, 3], geom, use_bn)
- self.header = Header(use_bn)
-
- for m in self.modules():
- if isinstance(m, nn.Conv2d):
- n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
- m.weight.data.normal_(0, math.sqrt(2.0 / n))
- elif isinstance(m, nn.BatchNorm2d):
- m.weight.data.fill_(1)
- m.bias.data.zero_()
-
- prior = 0.01
- self.header.clshead.weight.data.fill_(-math.log((1.0 - prior) / prior))
- self.header.clshead.bias.data.fill_(0)
- self.header.reghead.weight.data.fill_(0)
- self.header.reghead.bias.data.fill_(0)
-
- def forward(self, data_dict):
- if "intermediate_features" in data_dict:
- return self.fuse_agents(data_dict)
-
- bev_input = data_dict["processed_lidar"]["bev_input"]
- record_len = data_dict["record_len"]
-
- features = self.backbone(bev_input, record_len)
- # cls -- (N, 1, W/4, L/4)
- # reg -- (N, 6, W/4, L/4)
- cls, reg = self.header(features)
-
- output_dict = {"cls": cls, "reg": reg}
-
- return output_dict
-
- def encode_agent(self, data_dict):
- """
- Encode one agent into PIXOR's three private backbone scales.
-
- Parameters
- ----------
- data_dict : dict
- Sender-local preprocessed BEV input.
-
- Returns
- -------
- dict
- Ordered ``c3``, ``c4``, and ``c5`` feature maps.
- """
- return {
- "feature_maps": self.backbone.encode(data_dict["processed_lidar"]["bev_input"]),
- }
-
- def fuse_agents(self, data_dict):
- """
- Fuse received PIXOR features and run its detection header.
-
- Parameters
- ----------
- data_dict : dict
- Receiver input containing learned features and ``record_len``.
-
- Returns
- -------
- dict
- PIXOR classification and regression maps.
- """
- intermediate_features = data_dict["intermediate_features"]
- feature_maps = tuple(intermediate_features[f"feature_{index}"] for index in range(3))
- features = self.backbone.fuse_agents(feature_maps, data_dict["record_len"])
- cls, reg = self.header(features)
- return {"cls": cls, "reg": reg}
diff --git a/OpenCOOD/opencood/models/point_pillar.py b/OpenCOOD/opencood/models/point_pillar.py
deleted file mode 100644
index 3b6d12e6..00000000
--- a/OpenCOOD/opencood/models/point_pillar.py
+++ /dev/null
@@ -1,39 +0,0 @@
-import torch.nn as nn
-
-
-from opencood.models.sub_modules.pillar_vfe import PillarVFE
-from opencood.models.sub_modules.point_pillar_scatter import PointPillarScatter
-from opencood.models.sub_modules.base_bev_backbone import BaseBEVBackbone
-
-
-class PointPillar(nn.Module):
- def __init__(self, args):
- super(PointPillar, self).__init__()
-
- # PIllar VFE
- self.pillar_vfe = PillarVFE(args["pillar_vfe"], num_point_features=4, voxel_size=args["voxel_size"], point_cloud_range=args["lidar_range"])
- self.scatter = PointPillarScatter(args["point_pillar_scatter"])
- self.backbone = BaseBEVBackbone(args["base_bev_backbone"], 64)
-
- self.cls_head = nn.Conv2d(128 * 3, args["anchor_number"], kernel_size=1)
- self.reg_head = nn.Conv2d(128 * 3, 7 * args["anchor_num"], kernel_size=1)
-
- def forward(self, data_dict):
- voxel_features = data_dict["processed_lidar"]["voxel_features"]
- voxel_coords = data_dict["processed_lidar"]["voxel_coords"]
- voxel_num_points = data_dict["processed_lidar"]["voxel_num_points"]
-
- batch_dict = {"voxel_features": voxel_features, "voxel_coords": voxel_coords, "voxel_num_points": voxel_num_points}
-
- batch_dict = self.pillar_vfe(batch_dict)
- batch_dict = self.scatter(batch_dict)
- batch_dict = self.backbone(batch_dict)
-
- spatial_features_2d = batch_dict["spatial_features_2d"]
-
- psm = self.cls_head(spatial_features_2d)
- rm = self.reg_head(spatial_features_2d)
-
- output_dict = {"psm": psm, "rm": rm}
-
- return output_dict
diff --git a/OpenCOOD/opencood/models/point_pillar_coalign.py b/OpenCOOD/opencood/models/point_pillar_coalign.py
deleted file mode 100644
index d0193851..00000000
--- a/OpenCOOD/opencood/models/point_pillar_coalign.py
+++ /dev/null
@@ -1,177 +0,0 @@
-"""
-This file only contains CoAlign's multiscale intermediate feature fusion.
-If you want build the agent-object pose graph to correct pose error,
-please refer to https://github.com/yifanlu0227/CoAlign
-"""
-
-import torch.nn as nn
-
-from opencood.models.communication_adapters.intermediate import SpatialFeatureCommunicationAdapter
-from opencood.models.sub_modules.pillar_vfe import PillarVFE
-from opencood.models.sub_modules.point_pillar_scatter import PointPillarScatter
-from opencood.models.sub_modules.res_bev_backbone import ResBEVBackbone
-from opencood.models.sub_modules.naive_compress import NaiveCompressor
-from opencood.models.sub_modules.downsample_conv import DownsampleConv
-from opencood.models.fuse_modules.coalign_fuse import Att_w_Warp, normalize_pairwise_tfm
-
-
-class PointPillarCoAlign(nn.Module):
- communication_adapter_class = SpatialFeatureCommunicationAdapter
-
- def __init__(self, args):
- super(PointPillarCoAlign, self).__init__()
-
- # PIllar VFE
- self.pillar_vfe = PillarVFE(args["pillar_vfe"], num_point_features=4, voxel_size=args["voxel_size"], point_cloud_range=args["lidar_range"])
- self.scatter = PointPillarScatter(args["point_pillar_scatter"])
- self.backbone = ResBEVBackbone(args["res_bev_backbone"], 64)
-
- self.voxel_size = args["voxel_size"]
- self.first_layer_stride = args["res_bev_backbone"]["layer_strides"][0]
-
- # multiscale fusion network modules
- self.fusion_net = nn.ModuleList()
- for i in range(len(args["res_bev_backbone"]["layer_nums"])):
- # If proj_first = True, no actual warping is performed
- self.fusion_net.append(Att_w_Warp(args["res_bev_backbone"]["num_filters"][i]))
- self.out_channel = sum(args["res_bev_backbone"]["num_upsample_filter"])
-
- self.compression = False
- if "compression" in args:
- self.compression = True
- self.naive_compressor = NaiveCompressor(args["res_bev_backbone"]["num_filters"][0], args["compression"])
-
- # used to downsample the feature map for efficient computation
- self.shrink_flag = False
- if "shrink_header" in args:
- self.shrink_flag = True
- self.shrink_conv = DownsampleConv(args["shrink_header"])
- self.out_channel = args["shrink_header"]["dim"][-1]
-
- self.cls_head = nn.Conv2d(self.out_channel, args["anchor_number"], kernel_size=1)
- self.reg_head = nn.Conv2d(self.out_channel, 7 * args["anchor_num"], kernel_size=1)
-
- def forward(self, data_dict):
- if "intermediate_features" in data_dict:
- return self.fuse_agents(data_dict)
-
- voxel_features = data_dict["processed_lidar"]["voxel_features"]
- voxel_coords = data_dict["processed_lidar"]["voxel_coords"]
- voxel_num_points = data_dict["processed_lidar"]["voxel_num_points"]
- record_len = data_dict["record_len"]
-
- batch_dict = {"voxel_features": voxel_features, "voxel_coords": voxel_coords, "voxel_num_points": voxel_num_points, "record_len": record_len}
-
- batch_dict = self.pillar_vfe(batch_dict)
- batch_dict = self.scatter(batch_dict)
-
- # get affine matrix for feature warping
- _, _, H0, W0 = batch_dict["spatial_features"].shape # original feature map shape H0, W0.
- normalized_affine_matrix = normalize_pairwise_tfm(data_dict["pairwise_t_matrix"], H0, W0, self.voxel_size[0])
-
- spatial_features = batch_dict["spatial_features"]
-
- # multiscale fusion
- # The first scale feature 'feature_list[0]' for transmission. Default 100*352*64*32/1000000 = 72.0896
- feature_list = []
- feature_list.append(self.backbone.get_layer_i_feature(spatial_features, layer_i=0))
- if self.compression:
- feature_list[0] = self.naive_compressor(feature_list[0])
- for i in range(1, self.backbone.num_levels):
- feature_list.append(self.backbone.get_layer_i_feature(feature_list[i - 1], layer_i=i))
-
- fused_feature_list = []
- for i, fuse_module in enumerate(self.fusion_net):
- fused_feature_list.append(fuse_module(feature_list[i], record_len, normalized_affine_matrix))
- fused_feature = self.backbone.decode_multiscale_feature(fused_feature_list)
-
- # downsample feature to reduce memory
- if self.shrink_flag:
- fused_feature = self.shrink_conv(fused_feature)
-
- psm = self.cls_head(fused_feature)
- rm = self.reg_head(fused_feature)
-
- output_dict = {"psm": psm, "rm": rm}
-
- return output_dict
-
- def encode_agent(self, data_dict):
- """
- Encode one agent up to CoAlign's first fusion scale.
-
- Parameters
- ----------
- data_dict : dict
- Sender-local preprocessed LiDAR input.
-
- Returns
- -------
- dict
- First-scale learned feature transmitted by CoAlign.
- """
- processed_lidar = data_dict["processed_lidar"]
- batch_dict = {
- "voxel_features": processed_lidar["voxel_features"],
- "voxel_coords": processed_lidar["voxel_coords"],
- "voxel_num_points": processed_lidar["voxel_num_points"],
- }
- batch_dict = self.pillar_vfe(batch_dict)
- batch_dict = self.scatter(batch_dict)
- spatial_features = self.backbone.get_layer_i_feature(
- batch_dict["spatial_features"],
- layer_i=0,
- )
- if self.compression:
- spatial_features = self.naive_compressor.encode(spatial_features)
- return {"spatial_features": spatial_features}
-
- def fuse_agents(self, data_dict):
- """
- Fuse delivered CoAlign features and run the detection heads.
-
- Parameters
- ----------
- data_dict : dict
- Receiver input containing learned features and pairwise poses.
-
- Returns
- -------
- dict
- Classification and regression maps.
- """
- first_scale = data_dict["intermediate_features"]["spatial_features"]
- if self.compression:
- first_scale = self.naive_compressor.decode(first_scale)
- _, _, height, width = first_scale.shape
- normalized_affine_matrix = normalize_pairwise_tfm(
- data_dict["pairwise_t_matrix"],
- height,
- width,
- self.voxel_size[0],
- downsample_rate=self.first_layer_stride,
- )
-
- feature_list = [first_scale]
- for level in range(1, self.backbone.num_levels):
- feature_list.append(
- self.backbone.get_layer_i_feature(
- feature_list[level - 1],
- layer_i=level,
- )
- )
- fused_feature_list = [
- fuse_module(
- feature_list[level],
- data_dict["record_len"],
- normalized_affine_matrix,
- )
- for level, fuse_module in enumerate(self.fusion_net)
- ]
- fused_feature = self.backbone.decode_multiscale_feature(fused_feature_list)
- if self.shrink_flag:
- fused_feature = self.shrink_conv(fused_feature)
- return {
- "psm": self.cls_head(fused_feature),
- "rm": self.reg_head(fused_feature),
- }
diff --git a/OpenCOOD/opencood/models/point_pillar_cobevt.py b/OpenCOOD/opencood/models/point_pillar_cobevt.py
deleted file mode 100644
index 7e10c49b..00000000
--- a/OpenCOOD/opencood/models/point_pillar_cobevt.py
+++ /dev/null
@@ -1,169 +0,0 @@
-import torch.nn as nn
-from einops import repeat
-
-from opencood.models.communication_adapters.intermediate import SpatialFeatureCommunicationAdapter
-from opencood.models.sub_modules.pillar_vfe import PillarVFE
-from opencood.models.sub_modules.point_pillar_scatter import PointPillarScatter
-from opencood.models.sub_modules.base_bev_backbone import BaseBEVBackbone
-from opencood.models.sub_modules.downsample_conv import DownsampleConv
-from opencood.models.sub_modules.naive_compress import NaiveCompressor
-from opencood.models.fuse_modules.swap_fusion_modules import SwapFusionEncoder
-from opencood.models.fuse_modules.fuse_utils import regroup
-
-
-class PointPillarCoBEVT(nn.Module):
- communication_adapter_class = SpatialFeatureCommunicationAdapter
-
- def __init__(self, args):
- super(PointPillarCoBEVT, self).__init__()
-
- self.max_cav = args["max_cav"]
- # PIllar VFE
- self.pillar_vfe = PillarVFE(args["pillar_vfe"], num_point_features=4, voxel_size=args["voxel_size"], point_cloud_range=args["lidar_range"])
- self.scatter = PointPillarScatter(args["point_pillar_scatter"])
- self.backbone = BaseBEVBackbone(args["base_bev_backbone"], 64)
- # used to downsample the feature map for efficient computation
- self.shrink_flag = False
- if "shrink_header" in args:
- self.shrink_flag = True
- self.shrink_conv = DownsampleConv(args["shrink_header"])
- self.compression = False
-
- if args["compression"] > 0:
- self.compression = True
- self.naive_compressor = NaiveCompressor(256, args["compression"])
-
- self.fusion_net = SwapFusionEncoder(args["fax_fusion"])
-
- self.cls_head = nn.Conv2d(128 * 2, args["anchor_number"], kernel_size=1)
- self.reg_head = nn.Conv2d(128 * 2, 7 * args["anchor_number"], kernel_size=1)
-
- if args["backbone_fix"]:
- self.backbone_fix()
-
- def backbone_fix(self):
- """
- Fix the parameters of backbone during finetune on timedelay。
- """
- for p in self.pillar_vfe.parameters():
- p.requires_grad = False
-
- for p in self.scatter.parameters():
- p.requires_grad = False
-
- for p in self.backbone.parameters():
- p.requires_grad = False
-
- if self.compression:
- for p in self.naive_compressor.parameters():
- p.requires_grad = False
- if self.shrink_flag:
- for p in self.shrink_conv.parameters():
- p.requires_grad = False
-
- for p in self.cls_head.parameters():
- p.requires_grad = False
- for p in self.reg_head.parameters():
- p.requires_grad = False
-
- def forward(self, data_dict):
- if "intermediate_features" in data_dict:
- return self.fuse_agents(data_dict)
-
- voxel_features = data_dict["processed_lidar"]["voxel_features"]
- voxel_coords = data_dict["processed_lidar"]["voxel_coords"]
- voxel_num_points = data_dict["processed_lidar"]["voxel_num_points"]
- record_len = data_dict["record_len"]
-
- batch_dict = {"voxel_features": voxel_features, "voxel_coords": voxel_coords, "voxel_num_points": voxel_num_points, "record_len": record_len}
- # n, 4 -> n, c
- batch_dict = self.pillar_vfe(batch_dict)
- # n, c -> N, C, H, W
- batch_dict = self.scatter(batch_dict)
- batch_dict = self.backbone(batch_dict)
-
- spatial_features_2d = batch_dict["spatial_features_2d"]
- # downsample feature to reduce memory
- if self.shrink_flag:
- spatial_features_2d = self.shrink_conv(spatial_features_2d)
- # compressor
- if self.compression:
- spatial_features_2d = self.naive_compressor(spatial_features_2d)
-
- # N, C, H, W -> B, L, C, H, W
- regroup_feature, mask = regroup(spatial_features_2d, record_len, self.max_cav)
- com_mask = mask.unsqueeze(1).unsqueeze(2).unsqueeze(3)
- com_mask = repeat(com_mask, "b h w c l -> b (h new_h) (w new_w) c l", new_h=regroup_feature.shape[3], new_w=regroup_feature.shape[4])
-
- fused_feature = self.fusion_net(regroup_feature, com_mask)
-
- psm = self.cls_head(fused_feature)
- rm = self.reg_head(fused_feature)
-
- output_dict = {"psm": psm, "rm": rm}
-
- return output_dict
-
- def encode_agent(self, data_dict):
- """
- Encode one agent into the feature map exchanged by CoBEVT.
-
- Parameters
- ----------
- data_dict : dict
- Sender-local preprocessed LiDAR input.
-
- Returns
- -------
- dict
- Learned spatial feature map before Swap Fusion.
- """
- processed_lidar = data_dict["processed_lidar"]
- batch_dict = {
- "voxel_features": processed_lidar["voxel_features"],
- "voxel_coords": processed_lidar["voxel_coords"],
- "voxel_num_points": processed_lidar["voxel_num_points"],
- }
- batch_dict = self.pillar_vfe(batch_dict)
- batch_dict = self.scatter(batch_dict)
- batch_dict = self.backbone(batch_dict)
- spatial_features = batch_dict["spatial_features_2d"]
- if self.shrink_flag:
- spatial_features = self.shrink_conv(spatial_features)
- if self.compression:
- spatial_features = self.naive_compressor.encode(spatial_features)
- return {"spatial_features": spatial_features}
-
- def fuse_agents(self, data_dict):
- """
- Fuse delivered CoBEVT features and run the detection heads.
-
- Parameters
- ----------
- data_dict : dict
- Receiver input containing learned features and ``record_len``.
-
- Returns
- -------
- dict
- Classification and regression maps.
- """
- spatial_features = data_dict["intermediate_features"]["spatial_features"]
- if self.compression:
- spatial_features = self.naive_compressor.decode(spatial_features)
- regroup_feature, mask = regroup(
- spatial_features,
- data_dict["record_len"],
- self.max_cav,
- )
- communication_mask = repeat(
- mask.unsqueeze(1).unsqueeze(2).unsqueeze(3),
- "b h w c l -> b (h new_h) (w new_w) c l",
- new_h=regroup_feature.shape[3],
- new_w=regroup_feature.shape[4],
- )
- fused_feature = self.fusion_net(regroup_feature, communication_mask)
- return {
- "psm": self.cls_head(fused_feature),
- "rm": self.reg_head(fused_feature),
- }
diff --git a/OpenCOOD/opencood/models/point_pillar_fcooper.py b/OpenCOOD/opencood/models/point_pillar_fcooper.py
deleted file mode 100644
index ff433c3a..00000000
--- a/OpenCOOD/opencood/models/point_pillar_fcooper.py
+++ /dev/null
@@ -1,158 +0,0 @@
-import torch.nn as nn
-
-from opencood.models.communication_adapters.intermediate import SpatialFeatureCommunicationAdapter
-from opencood.models.sub_modules.pillar_vfe import PillarVFE
-from opencood.models.sub_modules.point_pillar_scatter import PointPillarScatter
-from opencood.models.sub_modules.base_bev_backbone import BaseBEVBackbone
-from opencood.models.sub_modules.downsample_conv import DownsampleConv
-from opencood.models.sub_modules.naive_compress import NaiveCompressor
-from opencood.models.fuse_modules.f_cooper_fuse import SpatialFusion
-
-
-class PointPillarFCooper(nn.Module):
- """
- F-Cooper implementation with point pillar backbone.
- """
-
- communication_adapter_class = SpatialFeatureCommunicationAdapter
-
- def __init__(self, args):
- super(PointPillarFCooper, self).__init__()
-
- self.max_cav = args["max_cav"]
- # PIllar VFE
- self.pillar_vfe = PillarVFE(args["pillar_vfe"], num_point_features=4, voxel_size=args["voxel_size"], point_cloud_range=args["lidar_range"])
- self.scatter = PointPillarScatter(args["point_pillar_scatter"])
- self.backbone = BaseBEVBackbone(args["base_bev_backbone"], 64)
- # used to downsample the feature map for efficient computation
- self.shrink_flag = False
- if "shrink_header" in args:
- self.shrink_flag = True
- self.shrink_conv = DownsampleConv(args["shrink_header"])
- self.compression = False
-
- if args["compression"] > 0:
- self.compression = True
- self.naive_compressor = NaiveCompressor(256, args["compression"])
-
- self.fusion_net = SpatialFusion()
-
- self.cls_head = nn.Conv2d(128 * 2, args["anchor_number"], kernel_size=1)
- self.reg_head = nn.Conv2d(128 * 2, 7 * args["anchor_number"], kernel_size=1)
-
- if args["backbone_fix"]:
- self.backbone_fix()
-
- def backbone_fix(self):
- """
- Fix the parameters of backbone during finetune on timedelay。
- """
- for p in self.pillar_vfe.parameters():
- p.requires_grad = False
-
- for p in self.scatter.parameters():
- p.requires_grad = False
-
- for p in self.backbone.parameters():
- p.requires_grad = False
-
- if self.compression:
- for p in self.naive_compressor.parameters():
- p.requires_grad = False
- if self.shrink_flag:
- for p in self.shrink_conv.parameters():
- p.requires_grad = False
-
- for p in self.cls_head.parameters():
- p.requires_grad = False
- for p in self.reg_head.parameters():
- p.requires_grad = False
-
- def forward(self, data_dict):
- if "intermediate_features" in data_dict:
- return self.fuse_agents(data_dict)
-
- voxel_features = data_dict["processed_lidar"]["voxel_features"]
- voxel_coords = data_dict["processed_lidar"]["voxel_coords"]
- voxel_num_points = data_dict["processed_lidar"]["voxel_num_points"]
- record_len = data_dict["record_len"]
-
- batch_dict = {"voxel_features": voxel_features, "voxel_coords": voxel_coords, "voxel_num_points": voxel_num_points, "record_len": record_len}
- # n, 4 -> n, c
- batch_dict = self.pillar_vfe(batch_dict)
- # n, c -> N, C, H, W
- batch_dict = self.scatter(batch_dict)
- batch_dict = self.backbone(batch_dict)
-
- spatial_features_2d = batch_dict["spatial_features_2d"]
- # downsample feature to reduce memory
- if self.shrink_flag:
- spatial_features_2d = self.shrink_conv(spatial_features_2d)
- # compressor
- if self.compression:
- spatial_features_2d = self.naive_compressor(spatial_features_2d)
-
- fused_feature = self.fusion_net(spatial_features_2d, record_len)
-
- psm = self.cls_head(fused_feature)
- rm = self.reg_head(fused_feature)
-
- output_dict = {"psm": psm, "rm": rm}
-
- return output_dict
-
- def encode_agent(self, data_dict):
- """
- Encode one agent into the BEV feature exchanged by F-Cooper.
-
- Parameters
- ----------
- data_dict : dict
- Sender-local preprocessed LiDAR input.
-
- Returns
- -------
- dict
- Learned spatial feature map before F-Cooper fusion.
- """
- processed_lidar = data_dict["processed_lidar"]
- batch_dict = {
- "voxel_features": processed_lidar["voxel_features"],
- "voxel_coords": processed_lidar["voxel_coords"],
- "voxel_num_points": processed_lidar["voxel_num_points"],
- }
- batch_dict = self.pillar_vfe(batch_dict)
- batch_dict = self.scatter(batch_dict)
- batch_dict = self.backbone(batch_dict)
- spatial_features = batch_dict["spatial_features_2d"]
- if self.shrink_flag:
- spatial_features = self.shrink_conv(spatial_features)
- if self.compression:
- spatial_features = self.naive_compressor.encode(spatial_features)
- return {"spatial_features": spatial_features}
-
- def fuse_agents(self, data_dict):
- """
- Fuse delivered F-Cooper features and run the detection heads.
-
- Parameters
- ----------
- data_dict : dict
- Receiver input containing learned features and ``record_len``.
-
- Returns
- -------
- dict
- Classification and regression maps.
- """
- spatial_features = data_dict["intermediate_features"]["spatial_features"]
- if self.compression:
- spatial_features = self.naive_compressor.decode(spatial_features)
- fused_feature = self.fusion_net(
- spatial_features,
- data_dict["record_len"],
- )
- return {
- "psm": self.cls_head(fused_feature),
- "rm": self.reg_head(fused_feature),
- }
diff --git a/OpenCOOD/opencood/models/point_pillar_intermediate.py b/OpenCOOD/opencood/models/point_pillar_intermediate.py
deleted file mode 100644
index b0e625d2..00000000
--- a/OpenCOOD/opencood/models/point_pillar_intermediate.py
+++ /dev/null
@@ -1,93 +0,0 @@
-import torch.nn as nn
-
-from opencood.models.communication_adapters.intermediate import MultiScaleFeatureCommunicationAdapter
-from opencood.models.sub_modules.pillar_vfe import PillarVFE
-from opencood.models.sub_modules.point_pillar_scatter import PointPillarScatter
-from opencood.models.sub_modules.att_bev_backbone import AttBEVBackbone
-
-
-class PointPillarIntermediate(nn.Module):
- communication_adapter_class = MultiScaleFeatureCommunicationAdapter
-
- def __init__(self, args):
- super(PointPillarIntermediate, self).__init__()
-
- # PIllar VFE
- self.pillar_vfe = PillarVFE(args["pillar_vfe"], num_point_features=4, voxel_size=args["voxel_size"], point_cloud_range=args["lidar_range"])
- self.scatter = PointPillarScatter(args["point_pillar_scatter"])
- self.backbone = AttBEVBackbone(args["base_bev_backbone"], 64)
-
- self.cls_head = nn.Conv2d(128 * 3, args["anchor_number"], kernel_size=1)
- self.reg_head = nn.Conv2d(128 * 3, 7 * args["anchor_num"], kernel_size=1)
-
- def forward(self, data_dict):
- if "intermediate_features" in data_dict:
- return self.fuse_agents(data_dict)
-
- voxel_features = data_dict["processed_lidar"]["voxel_features"]
- voxel_coords = data_dict["processed_lidar"]["voxel_coords"]
- voxel_num_points = data_dict["processed_lidar"]["voxel_num_points"]
- record_len = data_dict["record_len"]
-
- batch_dict = {"voxel_features": voxel_features, "voxel_coords": voxel_coords, "voxel_num_points": voxel_num_points, "record_len": record_len}
-
- batch_dict = self.pillar_vfe(batch_dict)
- batch_dict = self.scatter(batch_dict)
- batch_dict = self.backbone(batch_dict)
-
- spatial_features_2d = batch_dict["spatial_features_2d"]
-
- psm = self.cls_head(spatial_features_2d)
- rm = self.reg_head(spatial_features_2d)
-
- output_dict = {"psm": psm, "rm": rm}
-
- return output_dict
-
- def encode_agent(self, data_dict):
- """
- Encode one agent into private multi-scale PointPillar features.
-
- Parameters
- ----------
- data_dict : dict
- Sender-local preprocessed LiDAR input.
-
- Returns
- -------
- dict
- Ordered feature maps at the model's attention boundaries.
- """
- processed_lidar = data_dict["processed_lidar"]
- batch_dict = {
- "voxel_features": processed_lidar["voxel_features"],
- "voxel_coords": processed_lidar["voxel_coords"],
- "voxel_num_points": processed_lidar["voxel_num_points"],
- }
- batch_dict = self.pillar_vfe(batch_dict)
- batch_dict = self.scatter(batch_dict)
- return {
- "feature_maps": self.backbone.encode_agent(batch_dict["spatial_features"]),
- }
-
- def fuse_agents(self, data_dict):
- """
- Fuse received feature scales and run PointPillar detection heads.
-
- Parameters
- ----------
- data_dict : dict
- Receiver input containing learned features and ``record_len``.
-
- Returns
- -------
- dict
- Classification and regression maps.
- """
- intermediate_features = data_dict["intermediate_features"]
- feature_maps = tuple(intermediate_features[f"feature_{index}"] for index in range(len(self.backbone.fuse_modules)))
- fused_features = self.backbone.fuse_agents(feature_maps, data_dict["record_len"])
- return {
- "psm": self.cls_head(fused_features),
- "rm": self.reg_head(fused_features),
- }
diff --git a/OpenCOOD/opencood/models/point_pillar_intermediate_V2VAM.py b/OpenCOOD/opencood/models/point_pillar_intermediate_V2VAM.py
deleted file mode 100644
index 16974c33..00000000
--- a/OpenCOOD/opencood/models/point_pillar_intermediate_V2VAM.py
+++ /dev/null
@@ -1,162 +0,0 @@
-import torch.nn as nn
-
-from opencood.models.communication_adapters.intermediate import SpatialFeatureCommunicationAdapter
-from opencood.models.sub_modules.pillar_vfe import PillarVFE
-from opencood.models.sub_modules.point_pillar_scatter import PointPillarScatter
-from opencood.models.sub_modules.base_bev_backbone import BaseBEVBackbone
-from opencood.models.sub_modules.downsample_conv import DownsampleConv
-from opencood.models.sub_modules.naive_compress import NaiveCompressor
-from opencood.models.fuse_modules.V2VAM import V2V_AttFusion
-
-
-class PointPillarintermediateV2VAM(nn.Module):
- """
- V2VAM implementation with a PointPillar backbone.
- """
-
- communication_adapter_class = SpatialFeatureCommunicationAdapter
-
- def __init__(self, args):
- super(PointPillarintermediateV2VAM, self).__init__()
-
- self.max_cav = args["max_cav"]
- # PIllar VFE
- self.pillar_vfe = PillarVFE(args["pillar_vfe"], num_point_features=4, voxel_size=args["voxel_size"], point_cloud_range=args["lidar_range"])
- self.scatter = PointPillarScatter(args["point_pillar_scatter"])
- self.backbone = BaseBEVBackbone(args["base_bev_backbone"], 64)
- # used to downsample the feature map for efficient computation
- self.shrink_flag = False
- if "shrink_header" in args:
- self.shrink_flag = True
- self.shrink_conv = DownsampleConv(args["shrink_header"])
- self.compression = False
-
- if args["compression"] > 0:
- self.compression = True
- print("self.compression: ", self.compression, args["compression"])
- self.naive_compressor = NaiveCompressor(256, args["compression"])
-
- self.fusion_net = V2V_AttFusion(256)
-
- self.cls_head = nn.Conv2d(128 * 2, args["anchor_number"], kernel_size=1)
- self.reg_head = nn.Conv2d(128 * 2, 7 * args["anchor_number"], kernel_size=1)
-
- if args["backbone_fix"]:
- self.backbone_fix()
-
- def backbone_fix(self):
- """
- Fix the parameters of backbone during finetune on timedelay。
- """
- for p in self.pillar_vfe.parameters():
- p.requires_grad = False
-
- for p in self.scatter.parameters():
- p.requires_grad = False
-
- for p in self.backbone.parameters():
- p.requires_grad = False
-
- if self.compression:
- for p in self.naive_compressor.parameters():
- p.requires_grad = False
- if self.shrink_flag:
- for p in self.shrink_conv.parameters():
- p.requires_grad = False
-
- for p in self.cls_head.parameters():
- p.requires_grad = False
- for p in self.reg_head.parameters():
- p.requires_grad = False
-
- def forward(self, data_dict):
- if "intermediate_features" in data_dict:
- return self.fuse_agents(data_dict)
-
- voxel_features = data_dict["processed_lidar"]["voxel_features"]
- voxel_coords = data_dict["processed_lidar"]["voxel_coords"]
- voxel_num_points = data_dict["processed_lidar"]["voxel_num_points"]
- record_len = data_dict["record_len"]
-
- batch_dict = {"voxel_features": voxel_features, "voxel_coords": voxel_coords, "voxel_num_points": voxel_num_points, "record_len": record_len}
- # n, 4 -> n, c
- batch_dict = self.pillar_vfe(batch_dict)
- # n, c -> N, C, H, W
- batch_dict = self.scatter(batch_dict)
- batch_dict = self.backbone(batch_dict)
-
- spatial_features_2d = batch_dict["spatial_features_2d"]
-
- # downsample feature to reduce memory
- if self.shrink_flag:
- spatial_features_2d = self.shrink_conv(spatial_features_2d)
- # compressor
- if self.compression:
- spatial_features_2d = self.naive_compressor(spatial_features_2d)
-
- fused_feature = self.fusion_net(spatial_features_2d, record_len)
-
- # pdb.set_trace()
-
- psm = self.cls_head(fused_feature)
- rm = self.reg_head(fused_feature)
-
- output_dict = {"psm": psm, "rm": rm}
-
- return output_dict
-
- def encode_agent(self, data_dict):
- """
- Encode one agent into the BEV feature exchanged by V2VAM.
-
- Parameters
- ----------
- data_dict : dict
- Sender-local preprocessed LiDAR input.
-
- Returns
- -------
- dict
- Learned spatial feature map before V2VAM fusion.
- """
- processed_lidar = data_dict["processed_lidar"]
- batch_dict = {
- "voxel_features": processed_lidar["voxel_features"],
- "voxel_coords": processed_lidar["voxel_coords"],
- "voxel_num_points": processed_lidar["voxel_num_points"],
- }
- batch_dict = self.pillar_vfe(batch_dict)
- batch_dict = self.scatter(batch_dict)
- batch_dict = self.backbone(batch_dict)
- spatial_features = batch_dict["spatial_features_2d"]
- if self.shrink_flag:
- spatial_features = self.shrink_conv(spatial_features)
- if self.compression:
- spatial_features = self.naive_compressor.encode(spatial_features)
- return {"spatial_features": spatial_features}
-
- def fuse_agents(self, data_dict):
- """
- Fuse delivered V2VAM features and run the detection heads.
-
- Parameters
- ----------
- data_dict : dict
- Receiver input containing learned features and ``record_len``.
-
- Returns
- -------
- dict
- Classification and regression maps.
- """
- spatial_features = data_dict["intermediate_features"]["spatial_features"]
- if self.compression:
- spatial_features = self.naive_compressor.decode(spatial_features)
- fused_feature = self.fusion_net(
- spatial_features,
- data_dict["record_len"],
- )
- return {
- "psm": self.cls_head(fused_feature),
- "rm": self.reg_head(fused_feature),
- }
diff --git a/OpenCOOD/opencood/models/point_pillar_transformer.py b/OpenCOOD/opencood/models/point_pillar_transformer.py
deleted file mode 100644
index a3649181..00000000
--- a/OpenCOOD/opencood/models/point_pillar_transformer.py
+++ /dev/null
@@ -1,185 +0,0 @@
-import torch
-import torch.nn as nn
-
-from opencood.models.communication_adapters.intermediate import SpatialFeatureCommunicationAdapter
-from opencood.models.sub_modules.pillar_vfe import PillarVFE
-from opencood.models.sub_modules.point_pillar_scatter import PointPillarScatter
-from opencood.models.sub_modules.base_bev_backbone import BaseBEVBackbone
-from opencood.models.fuse_modules.fuse_utils import regroup
-from opencood.models.sub_modules.downsample_conv import DownsampleConv
-from opencood.models.sub_modules.naive_compress import NaiveCompressor
-from opencood.models.fuse_modules.v2xvit_basic import V2XTransformer
-
-
-class PointPillarTransformer(nn.Module):
- communication_adapter_class = SpatialFeatureCommunicationAdapter
-
- def __init__(self, args):
- super(PointPillarTransformer, self).__init__()
-
- self.max_cav = args["max_cav"]
- # PIllar VFE
- self.pillar_vfe = PillarVFE(args["pillar_vfe"], num_point_features=4, voxel_size=args["voxel_size"], point_cloud_range=args["lidar_range"])
- self.scatter = PointPillarScatter(args["point_pillar_scatter"])
- self.backbone = BaseBEVBackbone(args["base_bev_backbone"], 64)
- # used to downsample the feature map for efficient computation
- self.shrink_flag = False
- if "shrink_header" in args:
- self.shrink_flag = True
- self.shrink_conv = DownsampleConv(args["shrink_header"])
- self.compression = False
-
- if args["compression"] > 0:
- self.compression = True
- self.naive_compressor = NaiveCompressor(256, args["compression"])
-
- self.fusion_net = V2XTransformer(args["transformer"])
-
- self.cls_head = nn.Conv2d(128 * 2, args["anchor_number"], kernel_size=1)
- self.reg_head = nn.Conv2d(128 * 2, 7 * args["anchor_number"], kernel_size=1)
-
- if args["backbone_fix"]:
- self.backbone_fix()
-
- def backbone_fix(self):
- """
- Fix the parameters of backbone during finetune on timedelay。
- """
- for p in self.pillar_vfe.parameters():
- p.requires_grad = False
-
- for p in self.scatter.parameters():
- p.requires_grad = False
-
- for p in self.backbone.parameters():
- p.requires_grad = False
-
- if self.compression:
- for p in self.naive_compressor.parameters():
- p.requires_grad = False
- if self.shrink_flag:
- for p in self.shrink_conv.parameters():
- p.requires_grad = False
-
- for p in self.cls_head.parameters():
- p.requires_grad = False
- for p in self.reg_head.parameters():
- p.requires_grad = False
-
- def forward(self, data_dict):
- if "intermediate_features" in data_dict:
- return self.fuse_agents(data_dict)
-
- voxel_features = data_dict["processed_lidar"]["voxel_features"]
- voxel_coords = data_dict["processed_lidar"]["voxel_coords"]
- voxel_num_points = data_dict["processed_lidar"]["voxel_num_points"]
- record_len = data_dict["record_len"]
- spatial_correction_matrix = data_dict["spatial_correction_matrix"]
-
- # B, max_cav, 3(dt dv infra), 1, 1
- prior_encoding = data_dict["prior_encoding"].unsqueeze(-1).unsqueeze(-1)
-
- batch_dict = {"voxel_features": voxel_features, "voxel_coords": voxel_coords, "voxel_num_points": voxel_num_points, "record_len": record_len}
- # n, 4 -> n, c
- batch_dict = self.pillar_vfe(batch_dict)
- # n, c -> N, C, H, W
- batch_dict = self.scatter(batch_dict)
- batch_dict = self.backbone(batch_dict)
-
- spatial_features_2d = batch_dict["spatial_features_2d"]
- # downsample feature to reduce memory
- if self.shrink_flag:
- spatial_features_2d = self.shrink_conv(spatial_features_2d)
- # compressor
- if self.compression:
- spatial_features_2d = self.naive_compressor(spatial_features_2d)
- # N, C, H, W -> B, L, C, H, W
- regroup_feature, mask = regroup(spatial_features_2d, record_len, self.max_cav)
- # prior encoding added
- prior_encoding = prior_encoding.repeat(1, 1, 1, regroup_feature.shape[3], regroup_feature.shape[4])
- regroup_feature = torch.cat([regroup_feature, prior_encoding], dim=2)
-
- # b l c h w -> b l h w c
- regroup_feature = regroup_feature.permute(0, 1, 3, 4, 2)
- # transformer fusion
- fused_feature = self.fusion_net(regroup_feature, mask, spatial_correction_matrix)
- # b h w c -> b c h w
- fused_feature = fused_feature.permute(0, 3, 1, 2)
-
- psm = self.cls_head(fused_feature)
- rm = self.reg_head(fused_feature)
-
- output_dict = {"psm": psm, "rm": rm}
-
- return output_dict
-
- def encode_agent(self, data_dict):
- """
- Encode one agent into the feature map exchanged by V2X-ViT.
-
- Parameters
- ----------
- data_dict : dict
- Sender-local preprocessed LiDAR input.
-
- Returns
- -------
- dict
- Learned spatial feature map before transformer fusion.
- """
- processed_lidar = data_dict["processed_lidar"]
- batch_dict = {
- "voxel_features": processed_lidar["voxel_features"],
- "voxel_coords": processed_lidar["voxel_coords"],
- "voxel_num_points": processed_lidar["voxel_num_points"],
- }
- batch_dict = self.pillar_vfe(batch_dict)
- batch_dict = self.scatter(batch_dict)
- batch_dict = self.backbone(batch_dict)
- spatial_features = batch_dict["spatial_features_2d"]
- if self.shrink_flag:
- spatial_features = self.shrink_conv(spatial_features)
- if self.compression:
- spatial_features = self.naive_compressor.encode(spatial_features)
- return {"spatial_features": spatial_features}
-
- def fuse_agents(self, data_dict):
- """
- Fuse delivered V2X-ViT features and run the detection heads.
-
- Parameters
- ----------
- data_dict : dict
- Receiver input containing learned features and V2X-ViT metadata.
-
- Returns
- -------
- dict
- Classification and regression maps.
- """
- spatial_features = data_dict["intermediate_features"]["spatial_features"]
- if self.compression:
- spatial_features = self.naive_compressor.decode(spatial_features)
- regroup_feature, mask = regroup(
- spatial_features,
- data_dict["record_len"],
- self.max_cav,
- )
- prior_encoding = data_dict["prior_encoding"].unsqueeze(-1).unsqueeze(-1)
- prior_encoding = prior_encoding.repeat(
- 1,
- 1,
- 1,
- regroup_feature.shape[3],
- regroup_feature.shape[4],
- )
- regroup_feature = torch.cat([regroup_feature, prior_encoding], dim=2)
- fused_feature = self.fusion_net(
- regroup_feature.permute(0, 1, 3, 4, 2),
- mask,
- data_dict["spatial_correction_matrix"],
- ).permute(0, 3, 1, 2)
- return {
- "psm": self.cls_head(fused_feature),
- "rm": self.reg_head(fused_feature),
- }
diff --git a/OpenCOOD/opencood/models/point_pillar_v2vnet.py b/OpenCOOD/opencood/models/point_pillar_v2vnet.py
deleted file mode 100644
index 715bf8eb..00000000
--- a/OpenCOOD/opencood/models/point_pillar_v2vnet.py
+++ /dev/null
@@ -1,156 +0,0 @@
-import torch.nn as nn
-
-from opencood.models.communication_adapters.intermediate import SpatialFeatureCommunicationAdapter
-from opencood.models.sub_modules.pillar_vfe import PillarVFE
-from opencood.models.sub_modules.point_pillar_scatter import PointPillarScatter
-from opencood.models.sub_modules.base_bev_backbone import BaseBEVBackbone
-from opencood.models.sub_modules.downsample_conv import DownsampleConv
-from opencood.models.sub_modules.naive_compress import NaiveCompressor
-from opencood.models.fuse_modules.v2v_fuse import V2VNetFusion
-
-
-class PointPillarV2VNet(nn.Module):
- communication_adapter_class = SpatialFeatureCommunicationAdapter
-
- def __init__(self, args):
- super(PointPillarV2VNet, self).__init__()
-
- self.max_cav = args["max_cav"]
- # PIllar VFE
- self.pillar_vfe = PillarVFE(args["pillar_vfe"], num_point_features=4, voxel_size=args["voxel_size"], point_cloud_range=args["lidar_range"])
- self.scatter = PointPillarScatter(args["point_pillar_scatter"])
- self.backbone = BaseBEVBackbone(args["base_bev_backbone"], 64)
-
- # used to downsample the feature map for efficient computation
- self.shrink_flag = False
- if "shrink_header" in args:
- self.shrink_flag = True
- self.shrink_conv = DownsampleConv(args["shrink_header"])
- self.compression = False
-
- if args["compression"] > 0:
- self.compression = True
- self.naive_compressor = NaiveCompressor(256, args["compression"])
-
- self.fusion_net = V2VNetFusion(args["v2vfusion"])
-
- self.cls_head = nn.Conv2d(128 * 2, args["anchor_number"], kernel_size=1)
- self.reg_head = nn.Conv2d(128 * 2, 7 * args["anchor_number"], kernel_size=1)
- if args["backbone_fix"]:
- self.backbone_fix()
-
- def backbone_fix(self):
- """
- Fix the parameters of backbone during finetune on timedelay。
- """
- for p in self.pillar_vfe.parameters():
- p.requires_grad = False
-
- for p in self.scatter.parameters():
- p.requires_grad = False
-
- for p in self.backbone.parameters():
- p.requires_grad = False
-
- if self.compression:
- for p in self.naive_compressor.parameters():
- p.requires_grad = False
- if self.shrink_flag:
- for p in self.shrink_conv.parameters():
- p.requires_grad = False
-
- for p in self.cls_head.parameters():
- p.requires_grad = False
- for p in self.reg_head.parameters():
- p.requires_grad = False
-
- def forward(self, data_dict):
- if "intermediate_features" in data_dict:
- return self.fuse_agents(data_dict)
-
- voxel_features = data_dict["processed_lidar"]["voxel_features"]
- voxel_coords = data_dict["processed_lidar"]["voxel_coords"]
- voxel_num_points = data_dict["processed_lidar"]["voxel_num_points"]
- record_len = data_dict["record_len"]
-
- pairwise_t_matrix = data_dict["pairwise_t_matrix"]
-
- batch_dict = {"voxel_features": voxel_features, "voxel_coords": voxel_coords, "voxel_num_points": voxel_num_points, "record_len": record_len}
- # n, 4 -> n, c
- batch_dict = self.pillar_vfe(batch_dict)
- # n, c -> N, C, H, W
- batch_dict = self.scatter(batch_dict)
- batch_dict = self.backbone(batch_dict)
-
- spatial_features_2d = batch_dict["spatial_features_2d"]
- # downsample feature to reduce memory
- if self.shrink_flag:
- spatial_features_2d = self.shrink_conv(spatial_features_2d)
- # compressor
- if self.compression:
- spatial_features_2d = self.naive_compressor(spatial_features_2d)
- fused_feature = self.fusion_net(spatial_features_2d, record_len, pairwise_t_matrix)
-
- psm = self.cls_head(fused_feature)
- rm = self.reg_head(fused_feature)
-
- output_dict = {"psm": psm, "rm": rm}
-
- return output_dict
-
- def encode_agent(self, data_dict):
- """
- Encode one agent into the BEV feature exchanged by V2VNet.
-
- Parameters
- ----------
- data_dict : dict
- Sender-local preprocessed LiDAR input.
-
- Returns
- -------
- dict
- Learned spatial feature map before V2VNet fusion.
- """
- processed_lidar = data_dict["processed_lidar"]
- batch_dict = {
- "voxel_features": processed_lidar["voxel_features"],
- "voxel_coords": processed_lidar["voxel_coords"],
- "voxel_num_points": processed_lidar["voxel_num_points"],
- }
- batch_dict = self.pillar_vfe(batch_dict)
- batch_dict = self.scatter(batch_dict)
- batch_dict = self.backbone(batch_dict)
- spatial_features = batch_dict["spatial_features_2d"]
- if self.shrink_flag:
- spatial_features = self.shrink_conv(spatial_features)
- if self.compression:
- spatial_features = self.naive_compressor.encode(spatial_features)
- return {"spatial_features": spatial_features}
-
- def fuse_agents(self, data_dict):
- """
- Fuse delivered V2VNet features and run the detection heads.
-
- Parameters
- ----------
- data_dict : dict
- Receiver input containing features, poses, and ``record_len``.
-
- Returns
- -------
- dict
- Classification and regression maps.
- """
- spatial_features = data_dict["intermediate_features"]["spatial_features"]
- if self.compression:
- spatial_features = self.naive_compressor.decode(spatial_features)
- fused_feature = self.fusion_net(
- spatial_features,
- data_dict["record_len"],
- data_dict["pairwise_t_matrix"],
- )
- return {
- "psm": self.cls_head(fused_feature),
- "rm": self.reg_head(fused_feature),
- }
diff --git a/OpenCOOD/opencood/models/point_pillar_where2comm.py b/OpenCOOD/opencood/models/point_pillar_where2comm.py
deleted file mode 100644
index 2b0f6de5..00000000
--- a/OpenCOOD/opencood/models/point_pillar_where2comm.py
+++ /dev/null
@@ -1,227 +0,0 @@
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-
-from opencood.models.communication_adapters.where2comm import Where2CommCommunicationAdapter
-from opencood.models.sub_modules.base_bev_backbone import BaseBEVBackbone
-from opencood.models.fuse_modules.where2comm_fuse import Where2comm
-from opencood.models.sub_modules.downsample_conv import DownsampleConv
-from opencood.models.sub_modules.naive_compress import NaiveCompressor
-from opencood.models.sub_modules.pillar_vfe import PillarVFE
-from opencood.models.sub_modules.point_pillar_scatter import PointPillarScatter
-
-
-class PointPillarWhere2comm(nn.Module):
- communication_adapter_class = Where2CommCommunicationAdapter
-
- def __init__(self, args):
- super(PointPillarWhere2comm, self).__init__()
- self.max_cav = args["max_cav"]
- # Pillar VFE
- self.pillar_vfe = PillarVFE(args["pillar_vfe"], num_point_features=4, voxel_size=args["voxel_size"], point_cloud_range=args["lidar_range"])
- self.scatter = PointPillarScatter(args["point_pillar_scatter"])
- self.backbone = BaseBEVBackbone(args["base_bev_backbone"], 64)
-
- # Used to down-sample the feature map for efficient computation
- if "shrink_header" in args:
- self.shrink_flag = True
- self.shrink_conv = DownsampleConv(args["shrink_header"])
- else:
- self.shrink_flag = False
-
- if args["compression"]:
- self.compression = True
- self.naive_compressor = NaiveCompressor(256, args["compression"])
- else:
- self.compression = False
-
- self.fusion_net = Where2comm(args["where2comm_fusion"])
- self.multi_scale = args["where2comm_fusion"]["multi_scale"]
-
- self.cls_head = nn.Conv2d(args["head_dim"], args["anchor_number"], kernel_size=1)
- self.reg_head = nn.Conv2d(args["head_dim"], 7 * args["anchor_number"], kernel_size=1)
-
- if args["backbone_fix"]:
- self.backbone_fix()
-
- def backbone_fix(self):
- """
- Fix the parameters of backbone during finetune on timedelay.
- """
-
- for p in self.pillar_vfe.parameters():
- p.requires_grad = False
-
- for p in self.scatter.parameters():
- p.requires_grad = False
-
- for p in self.backbone.parameters():
- p.requires_grad = False
-
- if self.compression:
- for p in self.naive_compressor.parameters():
- p.requires_grad = False
- if self.shrink_flag:
- for p in self.shrink_conv.parameters():
- p.requires_grad = False
-
- for p in self.cls_head.parameters():
- p.requires_grad = False
- for p in self.reg_head.parameters():
- p.requires_grad = False
-
- def forward(self, data_dict):
- if "intermediate_features" in data_dict:
- return self.fuse_agents(data_dict)
-
- voxel_features = data_dict["processed_lidar"]["voxel_features"]
- voxel_coords = data_dict["processed_lidar"]["voxel_coords"]
- voxel_num_points = data_dict["processed_lidar"]["voxel_num_points"]
- record_len = data_dict["record_len"]
- pairwise_t_matrix = data_dict["pairwise_t_matrix"]
-
- batch_dict = {"voxel_features": voxel_features, "voxel_coords": voxel_coords, "voxel_num_points": voxel_num_points, "record_len": record_len}
- # n, 4 -> n, c
- batch_dict = self.pillar_vfe(batch_dict)
- # n, c -> N, C, H, W
- batch_dict = self.scatter(batch_dict)
- batch_dict = self.backbone(batch_dict)
-
- # N, C, H', W': [N, 256, 48, 176]
- spatial_features_2d = batch_dict["spatial_features_2d"]
- # Down-sample feature to reduce memory
- if self.shrink_flag:
- spatial_features_2d = self.shrink_conv(spatial_features_2d)
-
- psm_single = self.cls_head(spatial_features_2d)
-
- # Compressor
- if self.compression:
- # The ego feature is also compressed
- spatial_features_2d = self.naive_compressor(spatial_features_2d)
-
- if self.multi_scale:
- # Bypass communication cost, communicate at high resolution, neither shrink nor compress
- fused_feature, communication_rates = self.fusion_net(
- batch_dict["spatial_features"], psm_single, record_len, pairwise_t_matrix, self.backbone
- )
- if self.shrink_flag:
- fused_feature = self.shrink_conv(fused_feature)
- else:
- fused_feature, communication_rates = self.fusion_net(spatial_features_2d, psm_single, record_len, pairwise_t_matrix)
-
- psm = self.cls_head(fused_feature)
- rm = self.reg_head(fused_feature)
-
- output_dict = {"psm": psm, "rm": rm, "com": communication_rates}
- return output_dict
-
- def encode_agent(self, data_dict, *, apply_communication_mask):
- """
- Encode one agent up to the Where2Comm transmission boundary.
-
- Parameters
- ----------
- data_dict : dict
- Sender-local preprocessed LiDAR input.
- apply_communication_mask : bool
- Whether to select only features intended for transmission.
-
- Returns
- -------
- dict
- Learned feature map and selected-location ratio.
- """
- processed_lidar = data_dict["processed_lidar"]
- batch_dict = {
- "voxel_features": processed_lidar["voxel_features"],
- "voxel_coords": processed_lidar["voxel_coords"],
- "voxel_num_points": processed_lidar["voxel_num_points"],
- "record_len": torch.ones(1, dtype=torch.long, device=processed_lidar["voxel_features"].device),
- }
- batch_dict = self.pillar_vfe(batch_dict)
- batch_dict = self.scatter(batch_dict)
- batch_dict = self.backbone(batch_dict)
-
- confidence_features = batch_dict["spatial_features_2d"]
- if self.shrink_flag:
- confidence_features = self.shrink_conv(confidence_features)
- confidence_map = self.cls_head(confidence_features)
-
- if self.multi_scale:
- spatial_features = self.backbone.blocks[0](batch_dict["spatial_features"])
- else:
- spatial_features = confidence_features
- if self.compression:
- spatial_features = self.naive_compressor.encode(spatial_features)
-
- communication_rate = torch.ones((), device=spatial_features.device)
- if not self.fusion_net.fully:
- communication_mask, communication_rate = self.fusion_net.naive_communication.build_mask(
- confidence_map,
- force_first=False,
- )
- if apply_communication_mask:
- if spatial_features.shape[-2:] != communication_mask.shape[-2:]:
- communication_mask = F.interpolate(
- communication_mask,
- size=spatial_features.shape[-2:],
- mode="bilinear",
- align_corners=False,
- )
- spatial_features = spatial_features * communication_mask
-
- return {
- "spatial_features": spatial_features,
- "communication_rate": communication_rate,
- }
-
- def fuse_agents(self, data_dict):
- """
- Fuse successfully received learned features and run detection heads.
-
- Parameters
- ----------
- data_dict : dict
- Receiver input containing encoded features and ``record_len``.
-
- Returns
- -------
- dict
- Fused classification, regression, and communication outputs.
- """
- intermediate_features = data_dict["intermediate_features"]
- spatial_features = intermediate_features["spatial_features"]
- record_len = data_dict["record_len"]
-
- if self.multi_scale:
- batch_node_features = self.fusion_net.regroup(spatial_features, record_len)
- fused_feature = torch.stack([self.fusion_net.fuse_modules[0](features) for features in batch_node_features])
- upsampled_features = [self.backbone.deblocks[0](fused_feature) if len(self.backbone.deblocks) > 0 else fused_feature]
-
- for level in range(1, self.fusion_net.num_levels):
- spatial_features = self.backbone.blocks[level](spatial_features)
- batch_node_features = self.fusion_net.regroup(spatial_features, record_len)
- fused_feature = torch.stack([self.fusion_net.fuse_modules[level](features) for features in batch_node_features])
- if len(self.backbone.deblocks) > 0:
- upsampled_features.append(self.backbone.deblocks[level](fused_feature))
- else:
- upsampled_features.append(fused_feature)
-
- fused_feature = torch.cat(upsampled_features, dim=1)
- if len(self.backbone.deblocks) > self.fusion_net.num_levels:
- fused_feature = self.backbone.deblocks[-1](fused_feature)
- if self.shrink_flag:
- fused_feature = self.shrink_conv(fused_feature)
- else:
- if self.compression:
- spatial_features = self.naive_compressor.decode(spatial_features)
- batch_node_features = self.fusion_net.regroup(spatial_features, record_len)
- fused_feature = torch.stack([self.fusion_net.fuse_modules(features) for features in batch_node_features])
-
- communication_rates = intermediate_features["communication_rate"]
- return {
- "psm": self.cls_head(fused_feature),
- "rm": self.reg_head(fused_feature),
- "com": communication_rates.mean(),
- }
diff --git a/OpenCOOD/opencood/models/second.py b/OpenCOOD/opencood/models/second.py
deleted file mode 100644
index 4f28467b..00000000
--- a/OpenCOOD/opencood/models/second.py
+++ /dev/null
@@ -1,51 +0,0 @@
-import torch.nn as nn
-
-from opencood.models.sub_modules.mean_vfe import MeanVFE
-from opencood.models.sub_modules.sparse_backbone_3d import VoxelBackBone8x
-from opencood.models.sub_modules.height_compression import HeightCompression
-from opencood.models.sub_modules.base_bev_backbone import BaseBEVBackbone
-
-
-class Second(nn.Module):
- def __init__(self, args):
- super(Second, self).__init__()
-
- self.batch_size = args["batch_size"]
- # mean_vfe
- self.mean_vfe = MeanVFE(args["mean_vfe"], 4)
- # sparse 3d backbone
- self.backbone_3d = VoxelBackBone8x(args["backbone_3d"], 4, args["grid_size"])
- # height compression
- self.height_compression = HeightCompression(args["height_compression"])
- # base ben backbone
- self.backbone_2d = BaseBEVBackbone(args["base_bev_backbone"], 256)
-
- # head
- self.cls_head = nn.Conv2d(256 * 2, args["anchor_number"], kernel_size=1)
- self.reg_head = nn.Conv2d(256 * 2, 7 * args["anchor_num"], kernel_size=1)
-
- def forward(self, data_dict):
- voxel_features = data_dict["processed_lidar"]["voxel_features"]
- voxel_coords = data_dict["processed_lidar"]["voxel_coords"]
- voxel_num_points = data_dict["processed_lidar"]["voxel_num_points"]
-
- batch_dict = {
- "voxel_features": voxel_features,
- "voxel_coords": voxel_coords,
- "voxel_num_points": voxel_num_points,
- "batch_size": data_dict.get("batch_size", self.batch_size),
- }
-
- batch_dict = self.mean_vfe(batch_dict)
- batch_dict = self.backbone_3d(batch_dict)
- batch_dict = self.height_compression(batch_dict)
- batch_dict = self.backbone_2d(batch_dict)
-
- spatial_features_2d = batch_dict["spatial_features_2d"]
-
- psm = self.cls_head(spatial_features_2d)
- rm = self.reg_head(spatial_features_2d)
-
- output_dict = {"psm": psm, "rm": rm}
-
- return output_dict
diff --git a/OpenCOOD/opencood/models/second_intermediate.py b/OpenCOOD/opencood/models/second_intermediate.py
deleted file mode 100644
index aad6f284..00000000
--- a/OpenCOOD/opencood/models/second_intermediate.py
+++ /dev/null
@@ -1,113 +0,0 @@
-import torch
-import torch.nn as nn
-
-from opencood.models.communication_adapters.intermediate import MultiScaleFeatureCommunicationAdapter
-from opencood.models.sub_modules.mean_vfe import MeanVFE
-from opencood.models.sub_modules.sparse_backbone_3d import VoxelBackBone8x
-from opencood.models.sub_modules.height_compression import HeightCompression
-from opencood.models.sub_modules.att_bev_backbone import AttBEVBackbone
-
-
-class SecondIntermediate(nn.Module):
- communication_adapter_class = MultiScaleFeatureCommunicationAdapter
-
- def __init__(self, args):
- super(SecondIntermediate, self).__init__()
-
- self.batch_size = args["batch_size"]
- # mean_vfe
- self.mean_vfe = MeanVFE(args["mean_vfe"], 4)
- # sparse 3d backbone
- self.backbone_3d = VoxelBackBone8x(args["backbone_3d"], 4, args["grid_size"])
- # height compression
- self.height_compression = HeightCompression(args["height_compression"])
- # base ben backbone
- self.backbone_2d = AttBEVBackbone(args["base_bev_backbone"], 256)
-
- # head
- self.cls_head = nn.Conv2d(256 * 2, args["anchor_number"], kernel_size=1)
- self.reg_head = nn.Conv2d(256 * 2, 7 * args["anchor_num"], kernel_size=1)
-
- def forward(self, data_dict):
- if "intermediate_features" in data_dict:
- return self.fuse_agents(data_dict)
-
- voxel_features = data_dict["processed_lidar"]["voxel_features"]
- voxel_coords = data_dict["processed_lidar"]["voxel_coords"]
- voxel_num_points = data_dict["processed_lidar"]["voxel_num_points"]
- record_len = data_dict["record_len"]
-
- batch_dict = {
- "voxel_features": voxel_features,
- "voxel_coords": voxel_coords,
- "voxel_num_points": voxel_num_points,
- "batch_size": torch.sum(record_len).cpu().numpy(),
- "record_len": record_len,
- }
-
- batch_dict = self.mean_vfe(batch_dict)
- batch_dict = self.backbone_3d(batch_dict)
- batch_dict = self.height_compression(batch_dict)
- batch_dict = self.backbone_2d(batch_dict)
-
- spatial_features_2d = batch_dict["spatial_features_2d"]
-
- psm = self.cls_head(spatial_features_2d)
- rm = self.reg_head(spatial_features_2d)
-
- output_dict = {"psm": psm, "rm": rm}
-
- return output_dict
-
- def encode_agent(self, data_dict):
- """
- Encode one agent into private multi-scale SECOND features.
-
- Parameters
- ----------
- data_dict : dict
- Sender-local preprocessed LiDAR input.
-
- Returns
- -------
- dict
- Ordered feature maps at the 2D attention boundaries.
- """
- processed_lidar = data_dict["processed_lidar"]
- batch_dict = {
- "voxel_features": processed_lidar["voxel_features"],
- "voxel_coords": processed_lidar["voxel_coords"],
- "voxel_num_points": processed_lidar["voxel_num_points"],
- "batch_size": data_dict.get("batch_size", 1),
- }
- batch_dict = self.mean_vfe(batch_dict)
- batch_dict = self.backbone_3d(batch_dict)
- batch_dict = self.height_compression(batch_dict)
- return {
- "feature_maps": self.backbone_2d.encode_agent(batch_dict["spatial_features"]),
- }
-
- def fuse_agents(self, data_dict):
- """
- Fuse received SECOND feature scales and run detection heads.
-
- Parameters
- ----------
- data_dict : dict
- Receiver input containing learned features and ``record_len``.
-
- Returns
- -------
- dict
- Classification and regression maps.
- """
- intermediate_features = data_dict["intermediate_features"]
- feature_maps = tuple(intermediate_features[f"feature_{index}"] for index in range(len(self.backbone_2d.fuse_modules)))
- fused_features = self.backbone_2d.fuse_agents(
- feature_maps,
- data_dict["record_len"],
- )
- return {
- "psm": self.cls_head(fused_features),
- "rm": self.reg_head(fused_features),
- }
diff --git a/OpenCOOD/opencood/models/sub_modules/att_bev_backbone.py b/OpenCOOD/opencood/models/sub_modules/att_bev_backbone.py
deleted file mode 100644
index b6ad7010..00000000
--- a/OpenCOOD/opencood/models/sub_modules/att_bev_backbone.py
+++ /dev/null
@@ -1,196 +0,0 @@
-import numpy as np
-import torch
-import torch.nn as nn
-
-from opencood.models.fuse_modules.self_attn import AttFusion
-from opencood.models.sub_modules.auto_encoder import AutoEncoder
-
-
-class AttBEVBackbone(nn.Module):
- def __init__(self, model_cfg, input_channels):
- super().__init__()
- self.model_cfg = model_cfg
- self.compress = False
-
- if "compression" in model_cfg and model_cfg["compression"] > 0:
- self.compress = True
- self.compress_layer = model_cfg["compression"]
-
- if "layer_nums" in self.model_cfg:
- assert len(self.model_cfg["layer_nums"]) == len(self.model_cfg["layer_strides"]) == len(self.model_cfg["num_filters"])
-
- layer_nums = self.model_cfg["layer_nums"]
- layer_strides = self.model_cfg["layer_strides"]
- num_filters = self.model_cfg["num_filters"]
- else:
- layer_nums = layer_strides = num_filters = []
-
- if "upsample_strides" in self.model_cfg:
- assert len(self.model_cfg["upsample_strides"]) == len(self.model_cfg["num_upsample_filter"])
-
- num_upsample_filters = self.model_cfg["num_upsample_filter"]
- upsample_strides = self.model_cfg["upsample_strides"]
-
- else:
- upsample_strides = num_upsample_filters = []
-
- num_levels = len(layer_nums)
- c_in_list = [input_channels, *num_filters[:-1]]
-
- self.blocks = nn.ModuleList()
- self.fuse_modules = nn.ModuleList()
- self.deblocks = nn.ModuleList()
-
- if self.compress:
- self.compression_modules = nn.ModuleList()
-
- for idx in range(num_levels):
- cur_layers = [
- nn.ZeroPad2d(1),
- nn.Conv2d(c_in_list[idx], num_filters[idx], kernel_size=3, stride=layer_strides[idx], padding=0, bias=False),
- nn.BatchNorm2d(num_filters[idx], eps=1e-3, momentum=0.01),
- nn.ReLU(),
- ]
-
- fuse_network = AttFusion(num_filters[idx])
- self.fuse_modules.append(fuse_network)
- if self.compress and self.compress_layer - idx > 0:
- self.compression_modules.append(AutoEncoder(num_filters[idx], self.compress_layer - idx))
-
- for k in range(layer_nums[idx]):
- cur_layers.extend(
- [
- nn.Conv2d(num_filters[idx], num_filters[idx], kernel_size=3, padding=1, bias=False),
- nn.BatchNorm2d(num_filters[idx], eps=1e-3, momentum=0.01),
- nn.ReLU(),
- ]
- )
-
- self.blocks.append(nn.Sequential(*cur_layers))
- if len(upsample_strides) > 0:
- stride = upsample_strides[idx]
- if stride >= 1:
- self.deblocks.append(
- nn.Sequential(
- nn.ConvTranspose2d(
- num_filters[idx], num_upsample_filters[idx], upsample_strides[idx], stride=upsample_strides[idx], bias=False
- ),
- nn.BatchNorm2d(num_upsample_filters[idx], eps=1e-3, momentum=0.01),
- nn.ReLU(),
- )
- )
- else:
- stride = np.round(1 / stride).astype(int)
- self.deblocks.append(
- nn.Sequential(
- nn.Conv2d(num_filters[idx], num_upsample_filters[idx], stride, stride=stride, bias=False),
- nn.BatchNorm2d(num_upsample_filters[idx], eps=1e-3, momentum=0.01),
- nn.ReLU(),
- )
- )
-
- c_in = sum(num_upsample_filters)
- if len(upsample_strides) > num_levels:
- self.deblocks.append(
- nn.Sequential(
- nn.ConvTranspose2d(c_in, c_in, upsample_strides[-1], stride=upsample_strides[-1], bias=False),
- nn.BatchNorm2d(c_in, eps=1e-3, momentum=0.01),
- nn.ReLU(),
- )
- )
-
- self.num_bev_features = c_in
-
- def forward(self, data_dict):
- spatial_features = data_dict["spatial_features"]
- record_len = data_dict["record_len"]
-
- upsampled_features = []
- features = spatial_features
- for level, block in enumerate(self.blocks):
- features = block(features)
- if self.compress and level < len(self.compression_modules):
- features = self.compression_modules[level](features)
- fused_features = self.fuse_modules[level](features, record_len)
- if len(self.deblocks) > 0:
- upsampled_features.append(self.deblocks[level](fused_features))
- else:
- upsampled_features.append(fused_features)
-
- if len(upsampled_features) > 1:
- features = torch.cat(upsampled_features, dim=1)
- elif len(upsampled_features) == 1:
- features = upsampled_features[0]
- else:
- raise ValueError("At least one backbone level is required")
-
- if len(self.deblocks) > len(self.blocks):
- features = self.deblocks[-1](features)
- data_dict["spatial_features_2d"] = features
- return data_dict
-
- def encode_agent(self, spatial_features):
- """
- Compute every private backbone scale before feature fusion.
-
- Parameters
- ----------
- spatial_features : torch.Tensor
- Unfused BEV features for one or more agents.
-
- Returns
- -------
- tuple[torch.Tensor, ...]
- Ordered private feature maps consumed by attention fusion.
- """
- feature_maps = []
- features = spatial_features
- for level, block in enumerate(self.blocks):
- features = block(features)
- if self.compress and level < len(self.compression_modules):
- encoded_features = self.compression_modules[level].encode(features)
- feature_maps.append(encoded_features)
- features = self.compression_modules[level].decode(encoded_features)
- else:
- feature_maps.append(features)
- return tuple(feature_maps)
-
- def fuse_agents(self, feature_maps, record_len):
- """
- Fuse encoded feature scales and decode the final BEV representation.
-
- Parameters
- ----------
- feature_maps : tuple[torch.Tensor, ...]
- Concatenated per-agent feature maps at every scale.
- record_len : torch.Tensor
- Number of successfully available agents in each sample.
-
- Returns
- -------
- torch.Tensor
- Decoded fused BEV feature map.
- """
- if len(feature_maps) != len(self.fuse_modules):
- raise ValueError("Unexpected number of intermediate feature scales")
-
- ups = []
- for level, features in enumerate(feature_maps):
- if self.compress and level < len(self.compression_modules):
- features = self.compression_modules[level].decode(features)
- fused_features = self.fuse_modules[level](features, record_len)
- if len(self.deblocks) > 0:
- ups.append(self.deblocks[level](fused_features))
- else:
- ups.append(fused_features)
-
- if len(ups) > 1:
- features = torch.cat(ups, dim=1)
- elif len(ups) == 1:
- features = ups[0]
- else:
- raise ValueError("At least one intermediate feature scale is required")
-
- if len(self.deblocks) > len(self.blocks):
- features = self.deblocks[-1](features)
- return features
diff --git a/OpenCOOD/opencood/models/sub_modules/auto_encoder.py b/OpenCOOD/opencood/models/sub_modules/auto_encoder.py
deleted file mode 100644
index c5719d04..00000000
--- a/OpenCOOD/opencood/models/sub_modules/auto_encoder.py
+++ /dev/null
@@ -1,91 +0,0 @@
-import torch.nn as nn
-
-
-class AutoEncoder(nn.Module):
- def __init__(self, feature_num, layer_num):
- super().__init__()
- self.feature_num = feature_num
- self.feature_stride = 2
-
- self.encoder = nn.ModuleList()
- self.decoder = nn.ModuleList()
-
- for i in range(layer_num):
- cur_layers = [
- nn.ZeroPad2d(1),
- nn.Conv2d(feature_num, feature_num, kernel_size=3, stride=2, padding=0, bias=False),
- nn.BatchNorm2d(feature_num, eps=1e-3, momentum=0.01),
- nn.ReLU(),
- ]
-
- cur_layers.extend(
- [
- nn.Conv2d(feature_num, feature_num // self.feature_stride, kernel_size=3, padding=1, bias=False),
- nn.BatchNorm2d(feature_num // self.feature_stride, eps=1e-3, momentum=0.01),
- nn.ReLU(),
- ]
- )
-
- self.encoder.append(nn.Sequential(*cur_layers))
- feature_num = feature_num // self.feature_stride
-
- feature_num = self.feature_num
- for i in range(layer_num):
- cur_layers = [
- nn.Sequential(
- nn.ConvTranspose2d(feature_num // 2, feature_num, kernel_size=2, stride=2, bias=False),
- nn.BatchNorm2d(feature_num, eps=1e-3, momentum=0.01),
- nn.ReLU(),
- )
- ]
-
- cur_layers.extend(
- [
- nn.Sequential(
- nn.Conv2d(feature_num, feature_num, kernel_size=3, stride=1, bias=False, padding=1),
- nn.BatchNorm2d(feature_num, eps=1e-3, momentum=0.01),
- nn.ReLU(),
- )
- ]
- )
- self.decoder.append(nn.Sequential(*cur_layers))
- feature_num //= 2
-
- def forward(self, x):
- return self.decode(self.encode(x))
-
- def encode(self, x):
- """
- Produce the compact spatial representation for transmission.
-
- Parameters
- ----------
- x : torch.Tensor
- Full-resolution feature map.
-
- Returns
- -------
- torch.Tensor
- Encoded feature map with reduced channels and resolution.
- """
- for i in range(len(self.encoder)):
- x = self.encoder[i](x)
- return x
-
- def decode(self, x):
- """
- Restore a compact spatial representation for feature fusion.
-
- Parameters
- ----------
- x : torch.Tensor
- Encoded feature map received over V2X.
-
- Returns
- -------
- torch.Tensor
- Decoded feature map matching the fusion network input.
- """
- for i in range(len(self.decoder) - 1, -1, -1):
- x = self.decoder[i](x)
- return x
diff --git a/OpenCOOD/opencood/models/sub_modules/base_bev_backbone.py b/OpenCOOD/opencood/models/sub_modules/base_bev_backbone.py
deleted file mode 100644
index 4b0f48b8..00000000
--- a/OpenCOOD/opencood/models/sub_modules/base_bev_backbone.py
+++ /dev/null
@@ -1,113 +0,0 @@
-import numpy as np
-import torch
-import torch.nn as nn
-
-
-class BaseBEVBackbone(nn.Module):
- def __init__(self, model_cfg, input_channels):
- super().__init__()
- self.model_cfg = model_cfg
-
- if "layer_nums" in self.model_cfg:
- assert len(self.model_cfg["layer_nums"]) == len(self.model_cfg["layer_strides"]) == len(self.model_cfg["num_filters"])
-
- layer_nums = self.model_cfg["layer_nums"]
- layer_strides = self.model_cfg["layer_strides"]
- num_filters = self.model_cfg["num_filters"]
- else:
- layer_nums = layer_strides = num_filters = []
-
- if "upsample_strides" in self.model_cfg:
- assert len(self.model_cfg["upsample_strides"]) == len(self.model_cfg["num_upsample_filter"])
-
- num_upsample_filters = self.model_cfg["num_upsample_filter"]
- upsample_strides = self.model_cfg["upsample_strides"]
-
- else:
- upsample_strides = num_upsample_filters = []
-
- num_levels = len(layer_nums)
- c_in_list = [input_channels, *num_filters[:-1]]
-
- self.blocks = nn.ModuleList()
- self.deblocks = nn.ModuleList()
-
- for idx in range(num_levels):
- cur_layers = [
- nn.ZeroPad2d(1),
- nn.Conv2d(c_in_list[idx], num_filters[idx], kernel_size=3, stride=layer_strides[idx], padding=0, bias=False),
- nn.BatchNorm2d(num_filters[idx], eps=1e-3, momentum=0.01),
- nn.ReLU(),
- ]
- for k in range(layer_nums[idx]):
- cur_layers.extend(
- [
- nn.Conv2d(num_filters[idx], num_filters[idx], kernel_size=3, padding=1, bias=False),
- nn.BatchNorm2d(num_filters[idx], eps=1e-3, momentum=0.01),
- nn.ReLU(),
- ]
- )
-
- self.blocks.append(nn.Sequential(*cur_layers))
- if len(upsample_strides) > 0:
- stride = upsample_strides[idx]
- if stride >= 1:
- self.deblocks.append(
- nn.Sequential(
- nn.ConvTranspose2d(
- num_filters[idx], num_upsample_filters[idx], upsample_strides[idx], stride=upsample_strides[idx], bias=False
- ),
- nn.BatchNorm2d(num_upsample_filters[idx], eps=1e-3, momentum=0.01),
- nn.ReLU(),
- )
- )
- else:
- stride = np.round(1 / stride).astype(int)
- self.deblocks.append(
- nn.Sequential(
- nn.Conv2d(num_filters[idx], num_upsample_filters[idx], stride, stride=stride, bias=False),
- nn.BatchNorm2d(num_upsample_filters[idx], eps=1e-3, momentum=0.01),
- nn.ReLU(),
- )
- )
-
- c_in = sum(num_upsample_filters)
- if len(upsample_strides) > num_levels:
- self.deblocks.append(
- nn.Sequential(
- nn.ConvTranspose2d(c_in, c_in, upsample_strides[-1], stride=upsample_strides[-1], bias=False),
- nn.BatchNorm2d(c_in, eps=1e-3, momentum=0.01),
- nn.ReLU(),
- )
- )
-
- self.num_bev_features = c_in
-
- def forward(self, data_dict):
- spatial_features = data_dict["spatial_features"]
-
- ups = []
- ret_dict = {}
- x = spatial_features
-
- for i in range(len(self.blocks)):
- x = self.blocks[i](x)
-
- stride = int(spatial_features.shape[2] / x.shape[2])
- ret_dict["spatial_features_%dx" % stride] = x
-
- if len(self.deblocks) > 0:
- ups.append(self.deblocks[i](x))
- else:
- ups.append(x)
-
- if len(ups) > 1:
- x = torch.cat(ups, dim=1)
- elif len(ups) == 1:
- x = ups[0]
-
- if len(self.deblocks) > len(self.blocks):
- x = self.deblocks[-1](x)
-
- data_dict["spatial_features_2d"] = x
- return data_dict
diff --git a/OpenCOOD/opencood/models/sub_modules/base_transformer.py b/OpenCOOD/opencood/models/sub_modules/base_transformer.py
deleted file mode 100644
index cf36927d..00000000
--- a/OpenCOOD/opencood/models/sub_modules/base_transformer.py
+++ /dev/null
@@ -1,78 +0,0 @@
-import torch
-from torch import nn
-
-from einops import rearrange
-
-
-class PreNormResidual(nn.Module):
- def __init__(self, dim, fn):
- super().__init__()
- self.norm = nn.LayerNorm(dim)
- self.fn = fn
-
- def forward(self, x, **kwargs):
- return self.fn(self.norm(x), **kwargs) + x
-
-
-class PreNorm(nn.Module):
- def __init__(self, dim, fn):
- super().__init__()
- self.norm = nn.LayerNorm(dim)
- self.fn = fn
-
- def forward(self, x, **kwargs):
- return self.fn(self.norm(x), **kwargs)
-
-
-class FeedForward(nn.Module):
- def __init__(self, dim, hidden_dim, dropout=0.0):
- super().__init__()
- self.net = nn.Sequential(nn.Linear(dim, hidden_dim), nn.GELU(), nn.Dropout(dropout), nn.Linear(hidden_dim, dim), nn.Dropout(dropout))
-
- def forward(self, x):
- return self.net(x)
-
-
-class CavAttention(nn.Module):
- """
- Vanilla CAV attention.
- """
-
- def __init__(self, dim, heads, dim_head=64, dropout=0.1):
- super().__init__()
- inner_dim = heads * dim_head
-
- self.heads = heads
- self.scale = dim_head**-0.5
-
- self.attend = nn.Softmax(dim=-1)
- self.to_qkv = nn.Linear(dim, inner_dim * 3, bias=False)
-
- self.to_out = nn.Sequential(nn.Linear(inner_dim, dim), nn.Dropout(dropout))
-
- def forward(self, x, mask, prior_encoding):
- # x: (B, L, H, W, C) -> (B, H, W, L, C)
- # mask: (B, L)
- x = x.permute(0, 2, 3, 1, 4)
- # mask: (B, 1, H, W, L, 1)
- mask = mask.unsqueeze(1)
-
- # qkv: [(B, H, W, L, C_inner) *3]
- qkv = self.to_qkv(x).chunk(3, dim=-1)
- # q: (B, M, H, W, L, C)
- q, k, v = map(lambda t: rearrange(t, "b h w l (m c) -> b m h w l c", m=self.heads), qkv)
-
- # attention, (B, M, H, W, L, L)
- att_map = torch.einsum("b m h w i c, b m h w j c -> b m h w i j", q, k) * self.scale
- # add mask
- att_map = att_map.masked_fill(mask == 0, -float("inf"))
- # softmax
- att_map = self.attend(att_map)
-
- # out:(B, M, H, W, L, C_head)
- out = torch.einsum("b m h w i j, b m h w j c -> b m h w i c", att_map, v)
- out = rearrange(out, "b m h w l c -> b h w l (m c)", m=self.heads)
- out = self.to_out(out)
- # (B L H W C)
- out = out.permute(0, 3, 1, 2, 4)
- return out
diff --git a/OpenCOOD/opencood/models/sub_modules/cia_ssd_utils.py b/OpenCOOD/opencood/models/sub_modules/cia_ssd_utils.py
deleted file mode 100644
index 66e56a79..00000000
--- a/OpenCOOD/opencood/models/sub_modules/cia_ssd_utils.py
+++ /dev/null
@@ -1,108 +0,0 @@
-import torch
-from torch import nn
-
-
-class SSFA(nn.Module):
- def __init__(self, args):
- super(SSFA, self).__init__()
- # 128
- self._num_input_features = args["feature_num"] # noqa: DC05
-
- seq = [nn.ZeroPad2d(1)] + get_conv_layers(
- "Conv2d", 128, 128, n_layers=3, kernel_size=[3, 3, 3], stride=[1, 1, 1], padding=[0, 1, 1], sequential=False
- )
- self.bottom_up_block_0 = nn.Sequential(*seq)
- self.bottom_up_block_1 = get_conv_layers("Conv2d", 128, 256, n_layers=3, kernel_size=[3, 3, 3], stride=[2, 1, 1], padding=[1, 1, 1])
-
- self.trans_0 = get_conv_layers("Conv2d", 128, 128, n_layers=1, kernel_size=[1], stride=[1], padding=[0])
- self.trans_1 = get_conv_layers("Conv2d", 256, 256, n_layers=1, kernel_size=[1], stride=[1], padding=[0])
-
- self.deconv_block_0 = get_conv_layers("ConvTranspose2d", 256, 128, n_layers=1, kernel_size=[3], stride=[2], padding=[1], output_padding=[1])
- self.deconv_block_1 = get_conv_layers("ConvTranspose2d", 256, 128, n_layers=1, kernel_size=[3], stride=[2], padding=[1], output_padding=[1])
-
- self.conv_0 = get_conv_layers("Conv2d", 128, 128, n_layers=1, kernel_size=[3], stride=[1], padding=[1])
- self.conv_1 = get_conv_layers("Conv2d", 128, 128, n_layers=1, kernel_size=[3], stride=[1], padding=[1])
-
- self.w_0 = get_conv_layers("Conv2d", 128, 1, n_layers=1, kernel_size=[1], stride=[1], padding=[0], relu_last=False)
- self.w_1 = get_conv_layers("Conv2d", 128, 1, n_layers=1, kernel_size=[1], stride=[1], padding=[0], relu_last=False)
-
- # default init_weights for conv(msra) and norm in ConvModule
- def init_weights(self):
- for m in self.modules():
- if isinstance(m, nn.Conv2d):
- nn.init.xavier_normal_(m.weight, gain=1)
- if hasattr(m, "bias") and m.bias is not None:
- nn.init.constant_(m.bias, 0)
-
- def forward(self, x):
- x_0 = self.bottom_up_block_0(x)
- x_1 = self.bottom_up_block_1(x_0)
- x_trans_0 = self.trans_0(x_0)
- x_trans_1 = self.trans_1(x_1)
- x_middle_0 = self.deconv_block_0(x_trans_1) + x_trans_0
- x_middle_1 = self.deconv_block_1(x_trans_1)
- x_output_0 = self.conv_0(x_middle_0)
- x_output_1 = self.conv_1(x_middle_1)
-
- x_weight_0 = self.w_0(x_output_0)
- x_weight_1 = self.w_1(x_output_1)
- x_weight = torch.softmax(torch.cat([x_weight_0, x_weight_1], dim=1), dim=1)
- x_output = x_output_0 * x_weight[:, 0:1, :, :] + x_output_1 * x_weight[:, 1:, :, :]
-
- return x_output.contiguous()
-
-
-def get_conv_layers(conv_name, in_channels, out_channels, n_layers, kernel_size, stride, padding, relu_last=True, sequential=True, **kwargs):
- """
- Build convolutional layers. kernel_size, stride and padding should be a list with the lengths that match n_layers
- """
- seq = []
- for i in range(n_layers):
- seq.extend(
- [
- getattr(nn, conv_name)(
- in_channels,
- out_channels,
- kernel_size[i],
- stride=stride[i],
- padding=padding[i],
- bias=False,
- **{k: v[i] for k, v in kwargs.items()},
- ),
- nn.BatchNorm2d(out_channels, eps=1e-3, momentum=0.01),
- ]
- )
- if i < n_layers - 1 or relu_last:
- seq.append(nn.ReLU())
- in_channels = out_channels
- if sequential:
- return nn.Sequential(*seq)
- else:
- return seq
-
-
-class Head(nn.Module):
- def __init__(self, num_input, num_pred, num_cls, num_iou=2, use_dir=False, num_dir=1):
- super(Head, self).__init__()
- self.use_dir = use_dir
-
- self.conv_box = nn.Conv2d(num_input, num_pred, 1) # 128 -> 14
- self.conv_cls = nn.Conv2d(num_input, num_cls, 1) # 128 -> 2
- self.conv_iou = nn.Conv2d(num_input, num_iou, 1, bias=False)
-
- if self.use_dir:
- self.conv_dir = nn.Conv2d(num_input, num_dir, 1) # 128 -> 4
-
- def forward(self, x):
- box_preds = self.conv_box(x)
- cls_preds = self.conv_cls(x)
- ret_dict = {"box_preds": box_preds, "cls_preds": cls_preds}
- if self.use_dir:
- dir_preds = self.conv_dir(x) # dir_preds.shape=[8, w, h, 4]
- ret_dict["dir_cls_preds"] = dir_preds
- else:
- ret_dict["dir_cls_preds"] = torch.zeros((len(box_preds), 1, 2))
-
- ret_dict["iou_preds"] = self.conv_iou(x)
-
- return ret_dict
diff --git a/OpenCOOD/opencood/models/sub_modules/convgru.py b/OpenCOOD/opencood/models/sub_modules/convgru.py
deleted file mode 100644
index 46b014bd..00000000
--- a/OpenCOOD/opencood/models/sub_modules/convgru.py
+++ /dev/null
@@ -1,191 +0,0 @@
-import torch
-from torch import nn
-from torch.autograd import Variable
-
-
-class ConvGRUCell(nn.Module):
- def __init__(self, input_size, input_dim, hidden_dim, kernel_size, bias):
- """
- Initialize the ConvLSTM cell
- :param input_size: (int, int)
- Height and width of input tensor as (height, width).
- :param input_dim: int
- Number of channels of input tensor.
- :param hidden_dim: int
- Number of channels of hidden state.
- :param kernel_size: (int, int)
- Size of the convolutional kernel.
- :param bias: bool
- Whether or not to add the bias.
- :param dtype: torch.cuda.FloatTensor or torch.FloatTensor
- Whether or not to use cuda.
- """
- super(ConvGRUCell, self).__init__()
- self.height, self.width = input_size
- self.padding = kernel_size[0] // 2, kernel_size[1] // 2
- self.hidden_dim = hidden_dim
- self.bias = bias
-
- self.conv_gates = nn.Conv2d(
- in_channels=input_dim + hidden_dim,
- out_channels=2 * self.hidden_dim,
- # for update_gate,reset_gate respectively
- kernel_size=kernel_size,
- padding=self.padding,
- bias=self.bias,
- )
-
- self.conv_can = nn.Conv2d(
- in_channels=input_dim + hidden_dim,
- out_channels=self.hidden_dim,
- # for candidate neural memory
- kernel_size=kernel_size,
- padding=self.padding,
- bias=self.bias,
- )
-
- def init_hidden(self, batch_size):
- return Variable(torch.zeros(batch_size, self.hidden_dim, self.height, self.width))
-
- def forward(self, input_tensor, h_cur):
- """
- :param self:
- :param input_tensor: (b, c, h, w)
- input is actually the target_model
- :param h_cur: (b, c_hidden, h, w)
- current hidden and cell states respectively
- :return: h_next,
- next hidden state
- """
- combined = torch.cat([input_tensor, h_cur], dim=1)
- combined_conv = self.conv_gates(combined)
-
- gamma, beta = torch.split(combined_conv, self.hidden_dim, dim=1)
- reset_gate = torch.sigmoid(gamma)
- update_gate = torch.sigmoid(beta)
-
- combined = torch.cat([input_tensor, reset_gate * h_cur], dim=1)
- cc_cnm = self.conv_can(combined)
- cnm = torch.tanh(cc_cnm)
-
- h_next = (1 - update_gate) * h_cur + update_gate * cnm
- return h_next
-
-
-class ConvGRU(nn.Module):
- def __init__(self, input_size, input_dim, hidden_dim, kernel_size, num_layers, batch_first=False, bias=True, return_all_layers=False):
- """
- :param input_size: (int, int)
- Height and width of input tensor as (height, width).
- :param input_dim: int e.g. 256
- Number of channels of input tensor.
- :param hidden_dim: int e.g. 1024
- Number of channels of hidden state.
- :param kernel_size: (int, int)
- Size of the convolutional kernel.
- :param num_layers: int
- Number of ConvLSTM layers
- :param dtype: torch.cuda.FloatTensor or torch.FloatTensor
- Whether or not to use cuda.
- :param alexnet_path: str
- pretrained alexnet parameters
- :param batch_first: bool
- if the first position of array is batch or not
- :param bias: bool
- Whether or not to add the bias.
- :param return_all_layers: bool
- if return hidden and cell states for all layers
- """
- super(ConvGRU, self).__init__()
-
- # Make sure that both `kernel_size` and
- # `hidden_dim` are lists having len == num_layers
- kernel_size = self._extend_for_multilayer(kernel_size, num_layers)
- hidden_dim = self._extend_for_multilayer(hidden_dim, num_layers)
- if not len(kernel_size) == len(hidden_dim) == num_layers:
- raise ValueError("Inconsistent list length.")
-
- self.height, self.width = input_size
- self.input_dim = input_dim
- self.hidden_dim = hidden_dim
- self.kernel_size = kernel_size
- self.num_layers = num_layers
- self.batch_first = batch_first
- self.bias = bias
- self.return_all_layers = return_all_layers
-
- cell_list = []
- for i in range(0, self.num_layers):
- cur_input_dim = input_dim if i == 0 else hidden_dim[i - 1]
- cell_list.append(
- ConvGRUCell(
- input_size=(self.height, self.width),
- input_dim=cur_input_dim,
- hidden_dim=self.hidden_dim[i],
- kernel_size=self.kernel_size[i],
- bias=self.bias,
- )
- )
-
- # convert python list to pytorch module
- self.cell_list = nn.ModuleList(cell_list)
-
- def forward(self, input_tensor, hidden_state=None):
- """
- :param input_tensor: (b, t, c, h, w) or (t,b,c,h,w)
- depends on if batch first or not extracted features from alexnet
- :param hidden_state:
- :return: layer_output_list, last_state_list
- """
- if not self.batch_first:
- # (t, b, c, h, w) -> (b, t, c, h, w)
- input_tensor = input_tensor.permute(1, 0, 2, 3, 4)
-
- # Implement stateful ConvLSTM
- if hidden_state is not None:
- raise NotImplementedError()
- else:
- hidden_state = self._init_hidden(batch_size=input_tensor.size(0), device=input_tensor.device, dtype=input_tensor.dtype)
-
- layer_output_list = []
- last_state_list = []
-
- seq_len = input_tensor.size(1)
- cur_layer_input = input_tensor
-
- for layer_idx in range(self.num_layers):
- h = hidden_state[layer_idx]
- output_inner = []
- for t in range(seq_len):
- # input current hidden and cell state
- # then compute the next hidden
- # and cell state through ConvLSTMCell forward function
- h = self.cell_list[layer_idx](
- input_tensor=cur_layer_input[:, t, :, :, :], # (b,t,c,h,w)
- h_cur=h,
- )
- output_inner.append(h)
-
- layer_output = torch.stack(output_inner, dim=1)
- cur_layer_input = layer_output
-
- layer_output_list.append(layer_output)
- last_state_list.append([h])
-
- if not self.return_all_layers:
- layer_output_list = layer_output_list[-1:]
- last_state_list = last_state_list[-1:]
-
- return layer_output_list, last_state_list
-
- def _init_hidden(self, batch_size, device=None, dtype=None):
- init_states = []
- for i in range(self.num_layers):
- init_states.append(self.cell_list[i].init_hidden(batch_size).to(device).to(dtype))
- return init_states
-
- @staticmethod
- def _extend_for_multilayer(param, num_layers):
- if not isinstance(param, list):
- param = [param] * num_layers
- return param
diff --git a/OpenCOOD/opencood/models/sub_modules/downsample_conv.py b/OpenCOOD/opencood/models/sub_modules/downsample_conv.py
deleted file mode 100644
index d4b45761..00000000
--- a/OpenCOOD/opencood/models/sub_modules/downsample_conv.py
+++ /dev/null
@@ -1,42 +0,0 @@
-"""
-Class used to downsample features by 3*3 conv
-"""
-
-import torch.nn as nn
-
-
-class DoubleConv(nn.Module):
- """
- Double convoltuion
- Args:
- in_channels: input channel num
- out_channels: output channel num
- """
-
- def __init__(self, in_channels, out_channels, kernel_size, stride, padding):
- super().__init__()
- self.double_conv = nn.Sequential(
- nn.Conv2d(in_channels, out_channels, kernel_size=kernel_size, stride=stride, padding=padding),
- nn.ReLU(inplace=True),
- nn.Conv2d(out_channels, out_channels, kernel_size=3, padding=1),
- nn.ReLU(inplace=True),
- )
-
- def forward(self, x):
- return self.double_conv(x)
-
-
-class DownsampleConv(nn.Module):
- def __init__(self, config):
- super(DownsampleConv, self).__init__()
- self.layers = nn.ModuleList([])
- input_dim = config["input_dim"]
-
- for ksize, dim, stride, padding in zip(config["kernal_size"], config["dim"], config["stride"], config["padding"]):
- self.layers.append(DoubleConv(input_dim, dim, kernel_size=ksize, stride=stride, padding=padding))
- input_dim = dim
-
- def forward(self, x):
- for i in range(len(self.layers)):
- x = self.layers[i](x)
- return x
diff --git a/OpenCOOD/opencood/models/sub_modules/height_compression.py b/OpenCOOD/opencood/models/sub_modules/height_compression.py
deleted file mode 100644
index e917a310..00000000
--- a/OpenCOOD/opencood/models/sub_modules/height_compression.py
+++ /dev/null
@@ -1,26 +0,0 @@
-import torch.nn as nn
-
-
-class HeightCompression(nn.Module):
- def __init__(self, model_cfg, **kwargs):
- super().__init__()
- self.model_cfg = model_cfg
- self.num_bev_features = self.model_cfg["feature_num"]
-
- def forward(self, batch_dict):
- """
- Args:
- batch_dict:
- encoded_spconv_tensor: sparse tensor
- Returns:
- batch_dict:
- spatial_features:
-
- """
- encoded_spconv_tensor = batch_dict["encoded_spconv_tensor"]
- spatial_features = encoded_spconv_tensor.dense()
- N, C, D, H, W = spatial_features.shape
- spatial_features = spatial_features.view(N, C * D, H, W)
- batch_dict["spatial_features"] = spatial_features
- batch_dict["spatial_features_stride"] = batch_dict["encoded_spconv_tensor_stride"]
- return batch_dict
diff --git a/OpenCOOD/opencood/models/sub_modules/matcher.py b/OpenCOOD/opencood/models/sub_modules/matcher.py
deleted file mode 100644
index e50c2353..00000000
--- a/OpenCOOD/opencood/models/sub_modules/matcher.py
+++ /dev/null
@@ -1,124 +0,0 @@
-import torch
-from torch import nn
-
-from opencood.pcdet_utils.iou3d_nms.iou3d_nms_utils import boxes_iou3d_gpu
-
-pi = 3.141592653
-
-
-def limit_period(val, offset=0.5, period=2 * pi):
- return val - torch.floor(val / period + offset) * period
-
-
-class Matcher(nn.Module):
- """Correct localization error and use Algorithm 1:
- BBox matching with scores to fuse the proposal BBoxes"""
-
- def __init__(self, cfg, pc_range):
- super(Matcher, self).__init__()
- self.pc_range = pc_range
-
- @torch.no_grad()
- def forward(self, data_dict):
- clusters, scores = self.clustering(data_dict)
- data_dict["boxes_fused"], data_dict["scores_fused"] = self.cluster_fusion(clusters, scores)
- self.merge_keypoints(data_dict)
- return data_dict
-
- def clustering(self, data_dict):
- """
- Assign predicted boxes to clusters according to their ious with each other
- """
- clusters_batch = []
- scores_batch = []
- record_len = [int(length) for length in data_dict["record_len"]]
- for i, length in enumerate(record_len):
- cur_boxes_list = data_dict["det_boxes"][sum(record_len[:i]) : sum(record_len[:i]) + length]
- cur_scores_list = data_dict["det_scores"][sum(record_len[:i]) : sum(record_len[:i]) + length]
- cur_boxes_list = [b for b in cur_boxes_list if len(b) > 0]
- cur_scores_list = [s for s in cur_scores_list if len(s) > 0]
- if len(cur_scores_list) == 0:
- clusters_batch.append([torch.Tensor([0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.57]).to(torch.device("cuda:0")).view(1, 7)])
- scores_batch.append([torch.Tensor([0.01]).to(torch.device("cuda:0")).view(-1)])
- continue
-
- pred_boxes_cat = torch.cat(cur_boxes_list, dim=0)
- pred_boxes_cat[:, -1] = limit_period(pred_boxes_cat[:, -1])
- pred_scores_cat = torch.cat(cur_scores_list, dim=0)
-
- ious = boxes_iou3d_gpu(pred_boxes_cat, pred_boxes_cat)
- cluster_indices = torch.zeros(len(ious)).int() # gt assignments of preds
- cur_cluster_id = 1
- while torch.any(cluster_indices == 0):
- cur_idx = torch.where(cluster_indices == 0)[0][0] # find the idx of the first pred which is not assigned yet
- cluster_indices[torch.where(ious[cur_idx] > 0.1)[0]] = cur_cluster_id
- cur_cluster_id += 1
- clusters = []
- scores = []
- for j in range(1, cur_cluster_id):
- clusters.append(pred_boxes_cat[cluster_indices == j])
- scores.append(pred_scores_cat[cluster_indices == j])
- clusters_batch.append(clusters)
- scores_batch.append(scores)
-
- return clusters_batch, scores_batch
-
- def cluster_fusion(self, clusters, scores):
- """
- Merge boxes in each cluster with scores as weights for merging
- """
- boxes_fused = []
- scores_fused = []
- for cl, sl in zip(clusters, scores):
- for c, s in zip(cl, sl):
- # reverse direction for non-dominant direction of boxes
- dirs = c[:, -1]
- max_score_idx = torch.argmax(s)
- dirs_diff = torch.abs(dirs - dirs[max_score_idx].item())
- lt_pi = (dirs_diff > pi).int()
- dirs_diff = dirs_diff * (1 - lt_pi) + (2 * pi - dirs_diff) * lt_pi
- score_lt_half_pi = s[dirs_diff > pi / 2].sum() # larger than
- score_set_half_pi = s[dirs_diff <= pi / 2].sum() # small equal than
- # select larger scored direction as final direction
- if score_lt_half_pi <= score_set_half_pi:
- dirs[dirs_diff > pi / 2] += pi
- else:
- dirs[dirs_diff <= pi / 2] += pi
- dirs = limit_period(dirs)
- s_normalized = s / s.sum()
- sint = torch.sin(dirs) * s_normalized
- cost = torch.cos(dirs) * s_normalized
- theta = torch.atan2(sint.sum(), cost.sum()).view(
- 1,
- )
- center_dim = c[:, :-1] * s_normalized[:, None]
- boxes_fused.append(torch.cat([center_dim.sum(dim=0), theta]))
- s_sorted = torch.sort(s, descending=True).values
- s_fused = 0
- for i, ss in enumerate(s_sorted):
- s_fused += ss ** (i + 1)
- s_fused = torch.tensor([min(s_fused, 1.0)], device=s.device)
- scores_fused.append(s_fused)
-
- assert len(boxes_fused) > 0
- boxes_fused = torch.stack(boxes_fused, dim=0)
- len_records = [len(c) for c in clusters]
- boxes_fused = [boxes_fused[sum(len_records[:i]) : sum(len_records[:i]) + length] for i, length in enumerate(len_records)]
- scores_fused = torch.stack(scores_fused, dim=0)
- scores_fused = [scores_fused[sum(len_records[:i]) : sum(len_records[:i]) + length] for i, length in enumerate(len_records)]
-
- return boxes_fused, scores_fused
-
- def merge_keypoints(self, data_dict):
- # merge keypoints
- kpts_feat_out = []
- kpts_coor_out = []
- keypoints_features = data_dict["point_features"]
- keypoints_coords = data_dict["point_coords"]
- idx = 0
- for length in data_dict["record_len"]:
- kpts_coor_out.append(torch.cat(keypoints_coords[idx : length + idx], dim=0))
- kpts_feat_out.append(torch.cat(keypoints_features[idx : length + idx], dim=0))
- idx += length
- data_dict["point_features"] = kpts_feat_out
- data_dict["point_coords"] = kpts_coor_out
diff --git a/OpenCOOD/opencood/models/sub_modules/mean_vfe.py b/OpenCOOD/opencood/models/sub_modules/mean_vfe.py
deleted file mode 100644
index 1fe4e8f5..00000000
--- a/OpenCOOD/opencood/models/sub_modules/mean_vfe.py
+++ /dev/null
@@ -1,28 +0,0 @@
-import torch
-import torch.nn as nn
-
-
-class MeanVFE(nn.Module):
- def __init__(self, model_cfg, num_point_features, **kwargs):
- super().__init__()
- self.model_cfg = model_cfg
- self.num_point_features = num_point_features
-
- def forward(self, batch_dict, **kwargs):
- """
- Args:
- batch_dict:
- voxels: (num_voxels, max_points_per_voxel, C)
- voxel_num_points: optional (num_voxels)
- **kwargs:
-
- Returns:
- vfe_features: (num_voxels, C)
- """
- voxel_features, voxel_num_points = batch_dict["voxel_features"], batch_dict["voxel_num_points"]
- points_mean = voxel_features[:, :, :].sum(dim=1, keepdim=False)
- normalizer = torch.clamp_min(voxel_num_points.view(-1, 1), min=1.0).type_as(voxel_features)
- points_mean = points_mean / normalizer
- batch_dict["voxel_features"] = points_mean.contiguous()
-
- return batch_dict
diff --git a/OpenCOOD/opencood/models/sub_modules/naive_compress.py b/OpenCOOD/opencood/models/sub_modules/naive_compress.py
deleted file mode 100644
index 8c2cfbd9..00000000
--- a/OpenCOOD/opencood/models/sub_modules/naive_compress.py
+++ /dev/null
@@ -1,58 +0,0 @@
-import torch.nn as nn
-
-
-class NaiveCompressor(nn.Module):
- """
- A very naive compression that only compress on the channel.
- """
-
- def __init__(self, input_dim, compress_raito):
- super().__init__()
- self.encoder = nn.Sequential(
- nn.Conv2d(input_dim, input_dim // compress_raito, kernel_size=3, stride=1, padding=1),
- nn.BatchNorm2d(input_dim // compress_raito, eps=1e-3, momentum=0.01),
- nn.ReLU(),
- )
- self.decoder = nn.Sequential(
- nn.Conv2d(input_dim // compress_raito, input_dim, kernel_size=3, stride=1, padding=1),
- nn.BatchNorm2d(input_dim, eps=1e-3, momentum=0.01),
- nn.ReLU(),
- nn.Conv2d(input_dim, input_dim, kernel_size=3, stride=1, padding=1),
- nn.BatchNorm2d(input_dim, eps=1e-3, momentum=0.01),
- nn.ReLU(),
- )
-
- def forward(self, x):
- return self.decode(self.encode(x))
-
- def encode(self, x):
- """
- Produce the channel-compressed representation for transmission.
-
- Parameters
- ----------
- x : torch.Tensor
- Full-channel spatial feature map.
-
- Returns
- -------
- torch.Tensor
- Channel-compressed feature map.
- """
- return self.encoder(x)
-
- def decode(self, x):
- """
- Restore a transmitted channel-compressed representation.
-
- Parameters
- ----------
- x : torch.Tensor
- Channel-compressed feature map.
-
- Returns
- -------
- torch.Tensor
- Feature map restored to the fusion network's channel count.
- """
- return self.decoder(x)
diff --git a/OpenCOOD/opencood/models/sub_modules/pillar_vfe.py b/OpenCOOD/opencood/models/sub_modules/pillar_vfe.py
deleted file mode 100644
index ca3c6a6b..00000000
--- a/OpenCOOD/opencood/models/sub_modules/pillar_vfe.py
+++ /dev/null
@@ -1,117 +0,0 @@
-"""
-Pillar VFE, credits to OpenPCDet.
-"""
-
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-
-
-class PFNLayer(nn.Module):
- def __init__(self, in_channels, out_channels, use_norm=True, last_layer=False):
- super().__init__()
-
- self.last_vfe = last_layer
- self.use_norm = use_norm
- if not self.last_vfe:
- out_channels = out_channels // 2
-
- if self.use_norm:
- self.linear = nn.Linear(in_channels, out_channels, bias=False)
- self.norm = nn.BatchNorm1d(out_channels, eps=1e-3, momentum=0.01)
- else:
- self.linear = nn.Linear(in_channels, out_channels, bias=True)
-
- self.part = 50000
-
- def forward(self, inputs):
- if inputs.shape[0] > self.part:
- # nn.Linear performs randomly when batch size is too large
- num_parts = inputs.shape[0] // self.part
- part_linear_out = [self.linear(inputs[num_part * self.part : (num_part + 1) * self.part]) for num_part in range(num_parts + 1)]
- x = torch.cat(part_linear_out, dim=0)
- else:
- x = self.linear(inputs)
- torch.backends.cudnn.enabled = False # noqa: DC05
- x = self.norm(x.permute(0, 2, 1)).permute(0, 2, 1) if self.use_norm else x
- torch.backends.cudnn.enabled = True # noqa: DC05
- x = F.relu(x)
- x_max = torch.max(x, dim=1, keepdim=True)[0]
-
- if self.last_vfe:
- return x_max
- else:
- x_repeat = x_max.repeat(1, inputs.shape[1], 1)
- x_concatenated = torch.cat([x, x_repeat], dim=2)
- return x_concatenated
-
-
-class PillarVFE(nn.Module):
- def __init__(self, model_cfg, num_point_features, voxel_size, point_cloud_range):
- super().__init__()
- self.model_cfg = model_cfg
-
- self.use_norm = self.model_cfg["use_norm"]
- self.with_distance = self.model_cfg["with_distance"]
-
- self.use_absolute_xyz = self.model_cfg["use_absolute_xyz"]
- num_point_features += 6 if self.use_absolute_xyz else 3
- if self.with_distance:
- num_point_features += 1
-
- self.num_filters = self.model_cfg["num_filters"]
- assert len(self.num_filters) > 0
- num_filters = [num_point_features] + list(self.num_filters)
-
- pfn_layers = []
- for i in range(len(num_filters) - 1):
- in_filters = num_filters[i]
- out_filters = num_filters[i + 1]
- pfn_layers.append(PFNLayer(in_filters, out_filters, self.use_norm, last_layer=(i >= len(num_filters) - 2)))
- self.pfn_layers = nn.ModuleList(pfn_layers)
-
- self.voxel_x = voxel_size[0]
- self.voxel_y = voxel_size[1]
- self.voxel_z = voxel_size[2]
- self.x_offset = self.voxel_x / 2 + point_cloud_range[0]
- self.y_offset = self.voxel_y / 2 + point_cloud_range[1]
- self.z_offset = self.voxel_z / 2 + point_cloud_range[2]
-
- @staticmethod
- def get_paddings_indicator(actual_num, max_num, axis=0):
- actual_num = torch.unsqueeze(actual_num, axis + 1)
- max_num_shape = [1] * len(actual_num.shape)
- max_num_shape[axis + 1] = -1
- max_num = torch.arange(max_num, dtype=torch.int, device=actual_num.device).view(max_num_shape)
- paddings_indicator = actual_num.int() > max_num
- return paddings_indicator
-
- def forward(self, batch_dict):
- voxel_features, voxel_num_points, coords = batch_dict["voxel_features"], batch_dict["voxel_num_points"], batch_dict["voxel_coords"]
- points_mean = voxel_features[:, :, :3].sum(dim=1, keepdim=True) / voxel_num_points.type_as(voxel_features).view(-1, 1, 1)
- f_cluster = voxel_features[:, :, :3] - points_mean
-
- f_center = torch.zeros_like(voxel_features[:, :, :3])
- f_center[:, :, 0] = voxel_features[:, :, 0] - (coords[:, 3].to(voxel_features.dtype).unsqueeze(1) * self.voxel_x + self.x_offset)
- f_center[:, :, 1] = voxel_features[:, :, 1] - (coords[:, 2].to(voxel_features.dtype).unsqueeze(1) * self.voxel_y + self.y_offset)
- f_center[:, :, 2] = voxel_features[:, :, 2] - (coords[:, 1].to(voxel_features.dtype).unsqueeze(1) * self.voxel_z + self.z_offset)
-
- if self.use_absolute_xyz:
- features = [voxel_features, f_cluster, f_center]
- else:
- features = [voxel_features[..., 3:], f_cluster, f_center]
-
- if self.with_distance:
- points_dist = torch.norm(voxel_features[:, :, :3], 2, 2, keepdim=True)
- features.append(points_dist)
- features = torch.cat(features, dim=-1)
-
- voxel_count = features.shape[1]
- mask = self.get_paddings_indicator(voxel_num_points, voxel_count, axis=0)
- mask = torch.unsqueeze(mask, -1).type_as(voxel_features)
- features *= mask
- for pfn in self.pfn_layers:
- features = pfn(features)
- features = features.squeeze()
- batch_dict["pillar_features"] = features
- return batch_dict
diff --git a/OpenCOOD/opencood/models/sub_modules/point_pillar_scatter.py b/OpenCOOD/opencood/models/sub_modules/point_pillar_scatter.py
deleted file mode 100644
index 12022ce3..00000000
--- a/OpenCOOD/opencood/models/sub_modules/point_pillar_scatter.py
+++ /dev/null
@@ -1,39 +0,0 @@
-import torch
-import torch.nn as nn
-
-
-class PointPillarScatter(nn.Module):
- def __init__(self, model_cfg):
- super().__init__()
-
- self.model_cfg = model_cfg
- self.num_bev_features = self.model_cfg["num_features"]
- self.nx, self.ny, self.nz = model_cfg["grid_size"]
- assert self.nz == 1
-
- def forward(self, batch_dict):
- pillar_features, coords = batch_dict["pillar_features"], batch_dict["voxel_coords"]
- batch_spatial_features = []
- batch_size = coords[:, 0].max().int().item() + 1
-
- for batch_idx in range(batch_size):
- spatial_feature = torch.zeros(
- self.num_bev_features, self.nz * self.nx * self.ny, dtype=pillar_features.dtype, device=pillar_features.device
- )
-
- batch_mask = coords[:, 0] == batch_idx
- this_coords = coords[batch_mask, :]
-
- indices = this_coords[:, 1] + this_coords[:, 2] * self.nx + this_coords[:, 3]
- indices = indices.type(torch.long)
-
- pillars = pillar_features[batch_mask, :]
- pillars = pillars.t()
- spatial_feature[:, indices] = pillars
- batch_spatial_features.append(spatial_feature)
-
- batch_spatial_features = torch.stack(batch_spatial_features, 0)
- batch_spatial_features = batch_spatial_features.view(batch_size, self.num_bev_features * self.nz, self.ny, self.nx)
- batch_dict["spatial_features"] = batch_spatial_features
-
- return batch_dict
diff --git a/OpenCOOD/opencood/models/sub_modules/res_bev_backbone.py b/OpenCOOD/opencood/models/sub_modules/res_bev_backbone.py
deleted file mode 100644
index 168d0803..00000000
--- a/OpenCOOD/opencood/models/sub_modules/res_bev_backbone.py
+++ /dev/null
@@ -1,123 +0,0 @@
-import numpy as np
-import torch
-import torch.nn as nn
-from opencood.models.sub_modules.resblock import ResNetLayers
-
-
-class ResBEVBackbone(nn.Module):
- def __init__(self, model_cfg, input_channels=64):
- super().__init__()
- self.model_cfg = model_cfg
-
- if "layer_nums" in self.model_cfg:
- assert len(self.model_cfg["layer_nums"]) == len(self.model_cfg["layer_strides"]) == len(self.model_cfg["num_filters"])
-
- layer_nums = self.model_cfg["layer_nums"]
- layer_strides = self.model_cfg["layer_strides"]
- num_filters = self.model_cfg["num_filters"]
- else:
- layer_nums = layer_strides = num_filters = []
-
- if "upsample_strides" in self.model_cfg:
- assert len(self.model_cfg["upsample_strides"]) == len(self.model_cfg["num_upsample_filter"])
-
- num_upsample_filters = self.model_cfg["num_upsample_filter"]
- upsample_strides = self.model_cfg["upsample_strides"]
-
- else:
- upsample_strides = num_upsample_filters = []
-
- self.resnet = ResNetLayers(layer_nums, layer_strides, num_filters, inplanes=input_channels)
-
- num_levels = len(layer_nums)
- self.num_levels = len(layer_nums)
- self.deblocks = nn.ModuleList()
-
- for idx in range(num_levels):
- if len(upsample_strides) > 0:
- stride = upsample_strides[idx]
- if stride >= 1:
- self.deblocks.append(
- nn.Sequential(
- nn.ConvTranspose2d(
- num_filters[idx], num_upsample_filters[idx], upsample_strides[idx], stride=upsample_strides[idx], bias=False
- ),
- nn.BatchNorm2d(num_upsample_filters[idx], eps=1e-3, momentum=0.01),
- nn.ReLU(),
- )
- )
- else:
- stride = np.round(1 / stride).astype(int)
- self.deblocks.append(
- nn.Sequential(
- nn.Conv2d(num_filters[idx], num_upsample_filters[idx], stride, stride=stride, bias=False),
- nn.BatchNorm2d(num_upsample_filters[idx], eps=1e-3, momentum=0.01),
- nn.ReLU(),
- )
- )
-
- c_in = sum(num_upsample_filters)
- if len(upsample_strides) > num_levels:
- self.deblocks.append(
- nn.Sequential(
- nn.ConvTranspose2d(c_in, c_in, upsample_strides[-1], stride=upsample_strides[-1], bias=False),
- nn.BatchNorm2d(c_in, eps=1e-3, momentum=0.01),
- nn.ReLU(),
- )
- )
-
- self.num_bev_features = c_in
-
- def forward(self, data_dict):
- """
- This can be used in single agent detection (late fusion)
-
- For intermediate fusion, use get_multiscale_feature and
- decode_multiscale_feature.
- """
- spatial_features = data_dict["spatial_features"]
-
- x = self.resnet(spatial_features) # tuple of features
- ups = []
-
- for i in range(self.num_levels):
- if len(self.deblocks) > 0:
- ups.append(self.deblocks[i](x[i]))
- else:
- ups.append(x[i])
-
- if len(ups) > 1:
- x = torch.cat(ups, dim=1)
- elif len(ups) == 1:
- x = ups[0]
-
- if len(self.deblocks) > self.num_levels:
- x = self.deblocks[-1](x)
-
- data_dict["spatial_features_2d"] = x
- return data_dict
-
- def decode_multiscale_feature(self, x):
- """
- after multiscale interemediate fusion
- """
- ups = []
- for i in range(self.num_levels):
- if len(self.deblocks) > 0:
- ups.append(self.deblocks[i](x[i]))
- else:
- ups.append(x[i])
- if len(ups) > 1:
- x = torch.cat(ups, dim=1)
- elif len(ups) == 1:
- x = ups[0]
-
- if len(self.deblocks) > self.num_levels:
- x = self.deblocks[-1](x)
- return x
-
- def get_layer_i_feature(self, spatial_features, layer_i):
- """
- before multiscale intermediate fusion
- """
- return eval(f"self.resnet.layer{layer_i}")(spatial_features) # tuple of features
diff --git a/OpenCOOD/opencood/models/sub_modules/resblock.py b/OpenCOOD/opencood/models/sub_modules/resblock.py
deleted file mode 100644
index 637d5dea..00000000
--- a/OpenCOOD/opencood/models/sub_modules/resblock.py
+++ /dev/null
@@ -1,146 +0,0 @@
-from torch import Tensor
-import torch.nn as nn
-from typing import Callable, List, Optional
-
-
-def conv3x3(in_planes: int, out_planes: int, stride: int = 1, groups: int = 1, dilation: int = 1) -> nn.Conv2d:
- """3x3 convolution with padding"""
- return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, padding=dilation, groups=groups, bias=False, dilation=dilation)
-
-
-def conv1x1(in_planes: int, out_planes: int, stride: int = 1) -> nn.Conv2d:
- """1x1 convolution"""
- return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, bias=False)
-
-
-class BasicBlock(nn.Module):
- expansion: int = 1
-
- def __init__(
- self,
- inplanes: int,
- planes: int,
- stride: int = 1,
- downsample: Optional[nn.Module] = None,
- groups: int = 1,
- base_width: int = 64,
- dilation: int = 1,
- norm_layer: Optional[Callable[..., nn.Module]] = None,
- ) -> None:
- super(BasicBlock, self).__init__()
- if norm_layer is None:
- norm_layer = nn.BatchNorm2d
- if groups != 1 or base_width != 64:
- raise ValueError("BasicBlock only supports groups=1 and base_width=64")
- if dilation > 1:
- raise NotImplementedError("Dilation > 1 not supported in BasicBlock")
- # Both self.conv1 and self.downsample layers downsample the input when stride != 1
- self.conv1 = conv3x3(inplanes, planes, stride)
- self.bn1 = norm_layer(planes)
- self.relu = nn.ReLU(inplace=True)
- self.conv2 = conv3x3(planes, planes)
- self.bn2 = norm_layer(planes)
- self.downsample = downsample
- self.stride = stride
-
- def forward(self, x: Tensor) -> Tensor:
- identity = x
-
- out = self.conv1(x)
- out = self.bn1(out)
- out = self.relu(out)
-
- out = self.conv2(out)
- out = self.bn2(out)
-
- if self.downsample is not None:
- identity = self.downsample(x)
-
- out += identity
- out = self.relu(out)
-
- return out
-
-
-class ResNetLayers(nn.Module):
- def __init__(
- self,
- layers: List[int], # number of block in one layer
- layer_strides: List[int], # stride after one layer
- num_filters: List[int], # feature dim
- zero_init_residual: bool = False,
- groups: int = 1,
- width_per_group: int = 64,
- replace_stride_with_dilation: Optional[List[bool]] = None,
- norm_layer: Optional[Callable[..., nn.Module]] = None,
- inplanes=64,
- ) -> None:
- super(ResNetLayers, self).__init__()
-
- if norm_layer is None:
- norm_layer = nn.BatchNorm2d
- self._norm_layer = norm_layer
-
- self.inplanes = inplanes
- self.dilation = 1
- if replace_stride_with_dilation is None:
- # each element in the tuple indicates if we should replace
- # the 2x2 stride with a dilated convolution instead
- replace_stride_with_dilation = [False, False, False]
- if len(replace_stride_with_dilation) != 3:
- raise ValueError("replace_stride_with_dilation should be None or a 3-element tuple, got {}".format(replace_stride_with_dilation))
- self.groups = groups
- self.base_width = width_per_group
- self.layernum = len(num_filters)
-
- for i in range(self.layernum):
- self.__setattr__(f"layer{i}", self._make_layer(num_filters[i], layers[i], stride=layer_strides[i]))
-
- for m in self.modules():
- if isinstance(m, nn.Conv2d):
- nn.init.kaiming_normal_(m.weight, mode="fan_out", nonlinearity="relu")
- elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)):
- nn.init.constant_(m.weight, 1)
- nn.init.constant_(m.bias, 0)
-
- # Zero-initialize the last BN in each residual branch,
- # so that the residual branch starts with zeros, and each residual block behaves like an identity.
- # This improves the model by 0.2~0.3% according to https://arxiv.org/abs/1706.02677
- if zero_init_residual:
- for m in self.modules():
- if isinstance(m, BasicBlock):
- nn.init.constant_(m.bn2.weight, 0)
-
- def _make_layer(self, planes: int, blocks: int, stride: int = 1, dilate: bool = False) -> nn.Sequential:
- block = BasicBlock
- norm_layer = self._norm_layer
- downsample = None
- previous_dilation = self.dilation
- if dilate:
- self.dilation *= stride
- stride = 1
- if stride != 1 or self.inplanes != planes * block.expansion:
- downsample = nn.Sequential(
- conv1x1(self.inplanes, planes * block.expansion, stride),
- norm_layer(planes * block.expansion),
- )
-
- layers = []
- # if stride != 1, the first block will downsample the feature map
- # plane is the feature dim
- # if Bottleneck, then the output dim is planes * block.expansion(4)
- layers.append(block(self.inplanes, planes, stride, downsample, self.groups, self.base_width, previous_dilation, norm_layer))
- self.inplanes = planes * block.expansion
- for _ in range(1, blocks):
- layers.append(block(self.inplanes, planes, groups=self.groups, base_width=self.base_width, dilation=self.dilation, norm_layer=norm_layer))
-
- return nn.Sequential(*layers)
-
- def forward(self, x: Tensor):
- interm_features = []
-
- for i in range(self.layernum):
- x = eval(f"self.layer{i}")(x)
- interm_features.append(x)
-
- return interm_features
diff --git a/OpenCOOD/opencood/models/sub_modules/roi_head.py b/OpenCOOD/opencood/models/sub_modules/roi_head.py
deleted file mode 100644
index 28e4f0e0..00000000
--- a/OpenCOOD/opencood/models/sub_modules/roi_head.py
+++ /dev/null
@@ -1,263 +0,0 @@
-import copy
-
-import torch.nn as nn
-import torch
-import numpy as np
-from opencood.pcdet_utils.pointnet2.pointnet2_stack import pointnet2_modules as pointnet2_stack_modules
-from opencood.utils import common_utils
-from opencood.pcdet_utils.iou3d_nms.iou3d_nms_utils import boxes_iou3d_gpu
-from opencood.utils import box_utils
-
-
-class RoIHead(nn.Module):
- def __init__(self, model_cfg):
- super().__init__()
- self.model_cfg = model_cfg
- input_channels = model_cfg["in_channels"]
- self.code_size = 7
-
- mlps = copy.copy(self.model_cfg["roi_grid_pool"]["mlps"])
- for k in range(len(mlps)):
- mlps[k] = [input_channels] + mlps[k]
-
- self.roi_grid_pool_layer = pointnet2_stack_modules.StackSAModuleMSG(
- radii=self.model_cfg["roi_grid_pool"]["pool_radius"],
- nsamples=self.model_cfg["roi_grid_pool"]["n_sample"],
- mlps=mlps,
- use_xyz=True,
- pool_method=self.model_cfg["roi_grid_pool"]["pool_method"],
- )
-
- grid_size = self.model_cfg["roi_grid_pool"]["grid_size"]
- self.grid_size = grid_size
- c_out = sum([x[-1] for x in mlps])
- pre_channel = grid_size * grid_size * grid_size * c_out
- fc_layers = [self.model_cfg["n_fc_neurons"]] * 2
- self.shared_fc_layers, pre_channel = self._make_fc_layers(pre_channel, fc_layers)
-
- self.cls_layers, pre_channel = self._make_fc_layers(pre_channel, fc_layers, output_channels=self.model_cfg["num_cls"])
- self.iou_layers, _ = self._make_fc_layers(pre_channel, fc_layers, output_channels=self.model_cfg["num_cls"])
- self.reg_layers, _ = self._make_fc_layers(pre_channel, fc_layers, output_channels=self.model_cfg["num_cls"] * 7)
-
- self._init_weights(weight_init="xavier")
-
- def _init_weights(self, weight_init="xavier"):
- if weight_init == "kaiming":
- init_func = nn.init.kaiming_normal_
- elif weight_init == "xavier":
- init_func = nn.init.xavier_normal_
- elif weight_init == "normal":
- init_func = nn.init.normal_
- else:
- raise NotImplementedError
-
- for m in self.modules():
- if isinstance(m, nn.Conv2d) or isinstance(m, nn.Conv1d):
- if weight_init == "normal":
- init_func(m.weight, mean=0, std=0.001)
- else:
- init_func(m.weight)
- if m.bias is not None:
- nn.init.constant_(m.bias, 0)
- nn.init.normal_(self.reg_layers[-1].weight, mean=0, std=0.001)
-
- def _make_fc_layers(self, input_channels, fc_list, output_channels=None):
- fc_layers = []
- pre_channel = input_channels
- for k in range(len(fc_list)):
- fc_layers.extend(
- [
- nn.Conv1d(pre_channel, fc_list[k], kernel_size=1, bias=False),
- # nn.BatchNorm1d(fc_list[k]),
- nn.ReLU(),
- ]
- )
- pre_channel = fc_list[k]
- if self.model_cfg["dp_ratio"] > 0:
- fc_layers.append(nn.Dropout(self.model_cfg["dp_ratio"]))
- if output_channels is not None:
- fc_layers.append(nn.Conv1d(pre_channel, output_channels, kernel_size=1, bias=True))
- fc_layers = nn.Sequential(*fc_layers)
- return fc_layers, pre_channel
-
- def get_global_grid_points_of_roi(self, rois):
- rois = rois.view(-1, rois.shape[-1])
- batch_size_rcnn = rois.shape[0]
-
- # (B, 6x6x6, 3)
- local_roi_grid_points = self.get_dense_grid_points(rois, batch_size_rcnn, self.grid_size)
- global_roi_grid_points = common_utils.rotate_points_along_z(local_roi_grid_points.clone(), rois[:, 6]).squeeze(dim=1)
- global_center = rois[:, 0:3].clone()
- global_roi_grid_points += global_center.unsqueeze(dim=1)
- return global_roi_grid_points, local_roi_grid_points
-
- @staticmethod
- def get_dense_grid_points(rois, batch_size_rcnn, grid_size):
- """
- Get the local coordinates of each grid point of a roi in the coordinate
- system of the roi(origin lies in the center of this roi.
- """
- faked_features = rois.new_ones((grid_size, grid_size, grid_size))
- dense_idx = torch.stack(torch.where(faked_features), dim=1) # (N, 3) [x_idx, y_idx, z_idx]
- dense_idx = dense_idx.repeat(batch_size_rcnn, 1, 1).float() # (B, 6x6x6, 3)
-
- local_roi_size = rois.view(batch_size_rcnn, -1)[:, 3:6]
- roi_grid_points = (dense_idx + 0.5) / grid_size * local_roi_size.unsqueeze(dim=1) - (local_roi_size.unsqueeze(dim=1) / 2) # (B, 6x6x6, 3)
- return roi_grid_points
-
- def assign_targets(self, batch_dict):
- batch_dict["rcnn_label_dict"] = {
- "rois": [],
- "gt_of_rois": [],
- "gt_of_rois_src": [],
- "cls_tgt": [],
- "reg_tgt": [],
- "iou_tgt": [],
- "rois_anchor": [],
- "record_len": [],
- }
- # pred_boxes = [boxes[:, [0, 1, 2, 5, 4, 3, 6]] for boxes in batch_dict['boxes_fused']]
- pred_boxes = batch_dict["boxes_fused"]
- gt_boxes = [b[m][:, [0, 1, 2, 5, 4, 3, 6]].float() for b, m in zip(batch_dict["object_bbx_center"], batch_dict["object_bbx_mask"].bool())]
- for rois, gts in zip(pred_boxes, gt_boxes):
- gts[:, -1] *= 1
- ious = boxes_iou3d_gpu(rois, gts)
- max_ious, gt_inds = ious.max(dim=1)
- gt_of_rois = gts[gt_inds]
- rcnn_labels = (max_ious > 0.3).float()
- mask = torch.logical_not(rcnn_labels.bool())
-
- # set negative samples back to rois, no correction in stage2 for them
- gt_of_rois[mask] = rois[mask]
- gt_of_rois_src = gt_of_rois.clone().detach()
-
- # canoical transformation
- roi_center = rois[:, 0:3]
- # TODO: roi_ry > 0 in pcdet
- roi_ry = rois[:, 6] % (2 * np.pi)
- gt_of_rois[:, 0:3] = gt_of_rois[:, 0:3] - roi_center
- gt_of_rois[:, 6] = gt_of_rois[:, 6] - roi_ry
-
- # transfer LiDAR coords to local coords
- gt_of_rois = common_utils.rotate_points_along_z(points=gt_of_rois.view(-1, 1, gt_of_rois.shape[-1]), angle=-roi_ry.view(-1)).view(
- -1, gt_of_rois.shape[-1]
- )
-
- # flip orientation if rois have opposite orientation
- heading_label = (
- gt_of_rois[:, 6] + (torch.div(torch.abs(gt_of_rois[:, 6].min()), (2 * np.pi), rounding_mode="trunc") + 1) * 2 * np.pi
- ) % (2 * np.pi) # 0 ~ 2pi
- opposite_flag = (heading_label > np.pi * 0.5) & (heading_label < np.pi * 1.5)
-
- # (0 ~ pi/2, 3pi/2 ~ 2pi)
- heading_label[opposite_flag] = (heading_label[opposite_flag] + np.pi) % (2 * np.pi)
- flag = heading_label > np.pi
- heading_label[flag] = heading_label[flag] - np.pi * 2 # (-pi/2, pi/2)
- heading_label = torch.clamp(heading_label, min=-np.pi / 2, max=np.pi / 2)
- gt_of_rois[:, 6] = heading_label
-
- # generate regression target
- rois_anchor = rois.clone().detach().view(-1, self.code_size)
- rois_anchor[:, 0:3] = 0
- rois_anchor[:, 6] = 0
-
- reg_targets = box_utils.box_encode(gt_of_rois.view(-1, self.code_size), rois_anchor)
-
- batch_dict["rcnn_label_dict"]["rois"].append(rois)
- batch_dict["rcnn_label_dict"]["gt_of_rois"].append(gt_of_rois)
- batch_dict["rcnn_label_dict"]["gt_of_rois_src"].append(gt_of_rois_src)
- batch_dict["rcnn_label_dict"]["cls_tgt"].append(rcnn_labels)
- batch_dict["rcnn_label_dict"]["reg_tgt"].append(reg_targets)
- batch_dict["rcnn_label_dict"]["iou_tgt"].append(max_ious)
- batch_dict["rcnn_label_dict"]["rois_anchor"].append(rois_anchor)
- batch_dict["rcnn_label_dict"]["record_len"].append(rois.shape[0])
-
- # cat list to tensor
- for k, v in batch_dict["rcnn_label_dict"].items():
- if k == "record_len":
- continue
- batch_dict["rcnn_label_dict"][k] = torch.cat(v, dim=0)
-
- return batch_dict
-
- def prepare_rois(self, batch_dict):
- """
- Prepare fused proposals for inference without ground truth.
-
- Parameters
- ----------
- batch_dict : dict
- Model state containing one ``boxes_fused`` tensor per scene.
-
- Returns
- -------
- dict
- Model state with the proposal fields required by RoI pooling and
- stage-two box decoding.
- """
- rois_by_scene = batch_dict["boxes_fused"]
- rois = torch.cat(rois_by_scene, dim=0)
- rois_anchor = rois.clone().detach().view(-1, self.code_size)
- rois_anchor[:, 0:3] = 0
- rois_anchor[:, 6] = 0
- batch_dict["rcnn_label_dict"] = {
- "rois": rois,
- "rois_anchor": rois_anchor,
- "record_len": [rois.shape[0] for rois in rois_by_scene],
- }
- return batch_dict
-
- def roi_grid_pool(self, batch_dict):
- batch_size = len(batch_dict["record_len"])
- rois = batch_dict["rcnn_label_dict"]["rois"]
- point_coords = batch_dict["point_coords"]
- point_features = batch_dict["point_features"]
- label_record_len = batch_dict["rcnn_label_dict"]["record_len"]
-
- point_features = torch.cat(point_features, dim=0)
- # (BxN, 6x6x6, 3)
- global_roi_grid_points, _ = self.get_global_grid_points_of_roi(rois)
- # (B, Nx6x6x6, 3)
- global_roi_grid_points = global_roi_grid_points.view(batch_size, -1, 3)
-
- xyz = torch.cat(point_coords, dim=0)
- xyz_batch_cnt = xyz.new_zeros(batch_size).int()
- for bs_idx in range(batch_size):
- xyz_batch_cnt[bs_idx] = len(point_coords[bs_idx])
- new_xyz = global_roi_grid_points.view(-1, 3)
- new_xyz_batch_cnt = xyz.new_zeros(batch_size).int()
- for bs_idx in range(batch_size):
- new_xyz_batch_cnt[bs_idx] = label_record_len[bs_idx] * self.grid_size**3
-
- _, pooled_features = self.roi_grid_pool_layer(
- xyz=xyz[:, :3].contiguous(),
- xyz_batch_cnt=xyz_batch_cnt,
- new_xyz=new_xyz[:, :3].contiguous(),
- new_xyz_batch_cnt=new_xyz_batch_cnt,
- features=point_features.contiguous(), # weighted point features
- ) # (M1 + M2 ..., C)
- # (BxN, 6x6x6, C)
- pooled_features = pooled_features.view(-1, self.grid_size**3, pooled_features.shape[-1])
-
- return pooled_features
-
- def forward(self, batch_dict):
- batch_dict = self.assign_targets(batch_dict) if self.training else self.prepare_rois(batch_dict)
- # RoI aware pooling
- pooled_features = self.roi_grid_pool(batch_dict) # (BxN, 6x6x6, C)
-
- batch_size_rcnn = pooled_features.shape[0]
- pooled_features = (
- pooled_features.permute(0, 2, 1).contiguous().view(batch_size_rcnn, -1, self.grid_size, self.grid_size, self.grid_size)
- ) # (BxN, C, 6, 6, 6)
- shared_features = self.shared_fc_layers(pooled_features.view(batch_size_rcnn, -1, 1))
- rcnn_cls = self.cls_layers(shared_features).transpose(1, 2).contiguous().squeeze(dim=1) # (B, 1 or 2)
- rcnn_iou = self.iou_layers(shared_features).transpose(1, 2).contiguous().squeeze(dim=1) # (B, 1)
- rcnn_reg = self.reg_layers(shared_features).transpose(1, 2).contiguous().squeeze(dim=1) # (B, C)
-
- batch_dict["fpvrcnn_out"] = {
- "rcnn_cls": rcnn_cls,
- "rcnn_iou": rcnn_iou,
- "rcnn_reg": rcnn_reg,
- }
- return batch_dict
diff --git a/OpenCOOD/opencood/models/sub_modules/sparse_backbone_3d.py b/OpenCOOD/opencood/models/sub_modules/sparse_backbone_3d.py
deleted file mode 100644
index ffa1f392..00000000
--- a/OpenCOOD/opencood/models/sub_modules/sparse_backbone_3d.py
+++ /dev/null
@@ -1,134 +0,0 @@
-from functools import partial
-
-import torch.nn as nn
-
-try: # spconv1
- from spconv import SparseSequential, SubMConv3d, SparseConv3d, SparseInverseConv3d, SparseConvTensor
-except ImportError: # spconv2
- from spconv.pytorch import SparseSequential, SubMConv3d, SparseConv3d, SparseInverseConv3d, SparseConvTensor
-
-
-def post_act_block(in_channels, out_channels, kernel_size, indice_key=None, stride=1, padding=0, conv_type="subm", norm_fn=None):
- if conv_type == "subm":
- conv = SubMConv3d(in_channels, out_channels, kernel_size, bias=False, indice_key=indice_key)
- elif conv_type == "spconv":
- conv = SparseConv3d(in_channels, out_channels, kernel_size, stride=stride, padding=padding, bias=False, indice_key=indice_key)
- elif conv_type == "inverseconv":
- conv = SparseInverseConv3d(in_channels, out_channels, kernel_size, indice_key=indice_key, bias=False)
- else:
- raise NotImplementedError
-
- m = SparseSequential(
- conv,
- norm_fn(out_channels),
- nn.ReLU(),
- )
-
- return m
-
-
-class VoxelBackBone8x(nn.Module):
- def __init__(self, model_cfg, input_channels, grid_size, **kwargs):
- super().__init__()
- self.model_cfg = model_cfg
- norm_fn = partial(nn.BatchNorm1d, eps=1e-3, momentum=0.01)
-
- self.sparse_shape = grid_size[::-1] + [1, 0, 0]
-
- self.conv_input = SparseSequential(
- SubMConv3d(input_channels, 16, 3, padding=1, bias=False, indice_key="subm1"),
- norm_fn(16),
- nn.ReLU(),
- )
- block = post_act_block
-
- self.conv1 = SparseSequential(
- block(16, 16, 3, norm_fn=norm_fn, padding=1, indice_key="subm1"),
- )
-
- self.conv2 = SparseSequential(
- # [1600, 1408, 41] <- [800, 704, 21]
- block(16, 32, 3, norm_fn=norm_fn, stride=2, padding=1, indice_key="spconv2", conv_type="spconv"),
- block(32, 32, 3, norm_fn=norm_fn, padding=1, indice_key="subm2"),
- block(32, 32, 3, norm_fn=norm_fn, padding=1, indice_key="subm2"),
- )
-
- self.conv3 = SparseSequential(
- # [800, 704, 21] <- [400, 352, 11]
- block(32, 64, 3, norm_fn=norm_fn, stride=2, padding=1, indice_key="spconv3", conv_type="spconv"),
- block(64, 64, 3, norm_fn=norm_fn, padding=1, indice_key="subm3"),
- block(64, 64, 3, norm_fn=norm_fn, padding=1, indice_key="subm3"),
- )
-
- self.conv4 = SparseSequential(
- # [400, 352, 11] <- [200, 176, 5]
- block(64, 64, 3, norm_fn=norm_fn, stride=2, padding=(0, 1, 1), indice_key="spconv4", conv_type="spconv"),
- block(64, 64, 3, norm_fn=norm_fn, padding=1, indice_key="subm4"),
- block(64, 64, 3, norm_fn=norm_fn, padding=1, indice_key="subm4"),
- )
-
- last_pad = 0
- if "num_features_out" in self.model_cfg:
- self.num_point_features = self.model_cfg["num_features_out"]
- else:
- self.num_point_features = 128
- self.conv_out = SparseSequential(
- # [200, 150, 5] -> [200, 150, 2]
- SparseConv3d(64, self.num_point_features, (3, 1, 1), stride=(2, 1, 1), padding=last_pad, bias=False, indice_key="spconv_down2"),
- norm_fn(self.num_point_features),
- nn.ReLU(),
- )
-
- self.backbone_channels = {"x_conv1": 16, "x_conv2": 32, "x_conv3": 64, "x_conv4": 64} # noqa: DC05
-
- def forward(self, batch_dict):
- """
- Args:
- batch_dict:
- batch_size: int
- vfe_features: (num_voxels, C)
- voxel_coords: (num_voxels, 4), [batch_idx, z_idx, y_idx, x_idx]
- Returns:
- batch_dict:
- encoded_spconv_tensor: sparse tensor
- """
- voxel_features, voxel_coords = batch_dict["voxel_features"], batch_dict["voxel_coords"]
- batch_size = batch_dict["batch_size"]
- input_sp_tensor = SparseConvTensor(
- features=voxel_features, indices=voxel_coords.int(), spatial_shape=self.sparse_shape, batch_size=batch_size
- )
-
- x = self.conv_input(input_sp_tensor)
-
- x_conv1 = self.conv1(x)
- x_conv2 = self.conv2(x_conv1)
- x_conv3 = self.conv3(x_conv2)
- x_conv4 = self.conv4(x_conv3)
-
- # for detection head
- # [200, 176, 5] -> [200, 176, 2]
- out = self.conv_out(x_conv4)
-
- batch_dict.update({"encoded_spconv_tensor": out, "encoded_spconv_tensor_stride": 8})
- batch_dict.update(
- {
- "multi_scale_3d_features": {
- "x_conv1": x_conv1,
- "x_conv2": x_conv2,
- "x_conv3": x_conv3,
- "x_conv4": x_conv4,
- }
- }
- )
- batch_dict.update(
- {
- "multi_scale_3d_strides": {
- "x_conv1": 1,
- "x_conv2": 2,
- "x_conv3": 4,
- "x_conv4": 8,
- }
- }
- )
-
- return batch_dict
diff --git a/OpenCOOD/opencood/models/sub_modules/split_attn.py b/OpenCOOD/opencood/models/sub_modules/split_attn.py
deleted file mode 100644
index 8341fdf2..00000000
--- a/OpenCOOD/opencood/models/sub_modules/split_attn.py
+++ /dev/null
@@ -1,63 +0,0 @@
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-
-
-class RadixSoftmax(nn.Module):
- def __init__(self, radix, cardinality):
- super(RadixSoftmax, self).__init__()
- self.radix = radix
- self.cardinality = cardinality
-
- def forward(self, x):
- # x: (B, L, 1, 1, 3C)
- batch = x.size(0)
- cav_num = x.size(1)
-
- if self.radix > 1:
- # x: (B, L, 1, 3, C)
- x = x.view(batch, cav_num, self.cardinality, self.radix, -1)
- x = F.softmax(x, dim=3)
- # B, 3LC
- x = x.reshape(batch, -1)
- else:
- x = torch.sigmoid(x)
- return x
-
-
-class SplitAttn(nn.Module):
- def __init__(self, input_dim):
- super(SplitAttn, self).__init__()
- self.input_dim = input_dim
-
- self.fc1 = nn.Linear(input_dim, input_dim, bias=False)
- self.bn1 = nn.LayerNorm(input_dim)
- self.act1 = nn.ReLU()
- self.fc2 = nn.Linear(input_dim, input_dim * 3, bias=False)
-
- self.rsoftmax = RadixSoftmax(3, 1)
-
- def forward(self, window_list):
- # window list: [(B, L, H, W, C) * 3]
- assert len(window_list) == 3, "only 3 windows are supported"
-
- sw, mw, bw = window_list[0], window_list[1], window_list[2]
- B, L = sw.shape[0], sw.shape[1]
-
- # global average pooling, B, L, H, W, C
- x_gap = sw + mw + bw
- # B, L, 1, 1, C
- x_gap = x_gap.mean((2, 3), keepdim=True)
- x_gap = self.act1(self.bn1(self.fc1(x_gap)))
- # B, L, 1, 1, 3C
- x_attn = self.fc2(x_gap)
- # B L 1 1 3C
- x_attn = self.rsoftmax(x_attn).view(B, L, 1, 1, -1)
-
- out = (
- sw * x_attn[:, :, :, :, 0 : self.input_dim]
- + mw * x_attn[:, :, :, :, self.input_dim : 2 * self.input_dim]
- + bw * x_attn[:, :, :, :, self.input_dim * 2 :]
- )
-
- return out
diff --git a/OpenCOOD/opencood/models/sub_modules/torch_transformation_utils.py b/OpenCOOD/opencood/models/sub_modules/torch_transformation_utils.py
deleted file mode 100644
index f7804ba8..00000000
--- a/OpenCOOD/opencood/models/sub_modules/torch_transformation_utils.py
+++ /dev/null
@@ -1,396 +0,0 @@
-"""
-torch_transformation_utils.py
-"""
-
-import os
-
-import torch
-import torch.nn.functional as F
-import numpy as np
-import matplotlib.pyplot as plt
-
-
-def get_roi_and_cav_mask(shape, cav_mask, spatial_correction_matrix, discrete_ratio, downsample_rate):
- """
- Get mask for the combination of cav_mask and rorated ROI mask.
- Parameters
- ----------
- shape : tuple
- Shape of (B, L, H, W, C).
- cav_mask : torch.Tensor
- Shape of (B, L).
- spatial_correction_matrix : torch.Tensor
- Shape of (B, L, 4, 4)
- discrete_ratio : float
- Discrete ratio.
- downsample_rate : float
- Downsample rate.
-
- Returns
- -------
- com_mask : torch.Tensor
- Combined mask with shape (B, H, W, L, 1).
-
- """
- B, L, H, W, C = shape
- C = 1
- # (B,L,4,4)
- dist_correction_matrix = get_discretized_transformation_matrix(spatial_correction_matrix, discrete_ratio, downsample_rate)
- # (B*L,2,3)
- T = get_transformation_matrix(dist_correction_matrix.reshape(-1, 2, 3), (H, W))
- # (B,L,1,H,W)
- roi_mask = get_rotated_roi((B, L, C, H, W), T)
- # (B,L,1,H,W)
- com_mask = combine_roi_and_cav_mask(roi_mask, cav_mask)
- # (B,H,W,1,L)
- com_mask = com_mask.permute(0, 3, 4, 2, 1)
- return com_mask
-
-
-def combine_roi_and_cav_mask(roi_mask, cav_mask):
- """
- Combine ROI mask and CAV mask
-
- Parameters
- ----------
- roi_mask : torch.Tensor
- Mask for ROI region after considering the spatial transformation/correction.
- cav_mask : torch.Tensor
- Mask for CAV to remove padded 0.
-
- Returns
- -------
- com_mask : torch.Tensor
- Combined mask.
- """
- # (B, L, 1, 1, 1)
- cav_mask = cav_mask.unsqueeze(2).unsqueeze(3).unsqueeze(4)
- # (B, L, C, H, W)
- cav_mask = cav_mask.expand(roi_mask.shape)
- # (B, L, C, H, W)
- com_mask = roi_mask * cav_mask
- return com_mask
-
-
-def get_rotated_roi(shape, correction_matrix):
- """
- Get rorated ROI mask.
-
- Parameters
- ----------
- shape : tuple
- Shape of (B,L,C,H,W).
- correction_matrix : torch.Tensor
- Correction matrix with shape (N,2,3).
-
- Returns
- -------
- roi_mask : torch.Tensor
- Roated ROI mask with shape (N,2,3).
-
- """
- B, L, C, H, W = shape
- # To reduce the computation, we only need to calculate the
- # mask for the first channel.
- # (B,L,1,H,W)
- x = torch.ones((B, L, 1, H, W)).to(correction_matrix.dtype).to(correction_matrix.device)
- # (B*L,1,H,W)
- roi_mask = warp_affine(x.reshape(-1, 1, H, W), correction_matrix, dsize=(H, W), mode="nearest")
- # (B,L,C,H,W)
- roi_mask = torch.repeat_interleave(roi_mask, C, dim=1).reshape(B, L, C, H, W)
- return roi_mask
-
-
-def get_discretized_transformation_matrix(matrix, discrete_ratio, downsample_rate):
- """
- Get disretized transformation matrix.
- Parameters
- ----------
- matrix : torch.Tensor
- Shape -- (B, L, 4, 4) where B is the batch size, L is the max cav
- number.
- discrete_ratio : float
- Discrete ratio.
- downsample_rate : float/int
- downsample_rate
-
- Returns
- -------
- matrix : torch.Tensor
- Output transformation matrix in 2D with shape (B, L, 2, 3),
- including 2D transformation and 2D rotation.
-
- """
- matrix = matrix[:, :, [0, 1], :][:, :, :, [0, 1, 3]]
- # normalize the x,y transformation
- matrix[:, :, :, -1] = matrix[:, :, :, -1] / (discrete_ratio * downsample_rate)
-
- return matrix.type(dtype=torch.float)
-
-
-def _torch_inverse_cast(input):
- r"""
- Helper function to make torch.inverse work with other than fp32/64.
- The function torch.inverse is only implemented for fp32/64 which makes
- impossible to be used by fp16 or others. What this function does,
- is cast input data type to fp32, apply torch.inverse,
- and cast back to the input dtype.
- Args:
- input : torch.Tensor
- Tensor to be inversed.
-
- Returns:
- out : torch.Tensor
- Inversed Tensor.
-
- """
- dtype = input.dtype
- if dtype not in (torch.float32, torch.float64):
- dtype = torch.float32
- out = torch.inverse(input.to(dtype)).to(input.dtype)
- return out
-
-
-def normal_transform_pixel(height, width, device, dtype, eps=1e-14):
- r"""
- Compute the normalization matrix from image size in pixels to [-1, 1].
- Args:
- height : int
- Image height.
- width : int
- Image width.
- device : torch.device
- Output tensor devices.
- dtype : torch.dtype
- Output tensor data type.
- eps : float
- Epsilon to prevent divide-by-zero errors.
-
- Returns:
- tr_mat : torch.Tensor
- Normalized transform with shape :math:`(1, 3, 3)`.
- """
- tr_mat = torch.tensor([[1.0, 0.0, -1.0], [0.0, 1.0, -1.0], [0.0, 0.0, 1.0]], device=device, dtype=dtype) # 3x3
-
- # prevent divide by zero bugs
- width_denom = eps if width == 1 else width - 1.0
- height_denom = eps if height == 1 else height - 1.0
-
- tr_mat[0, 0] = tr_mat[0, 0] * 2.0 / width_denom
- tr_mat[1, 1] = tr_mat[1, 1] * 2.0 / height_denom
-
- return tr_mat.unsqueeze(0) # 1x3x3
-
-
-def eye_like(n, B, device, dtype):
- r"""
- Return a 2-D tensor with ones on the diagonal and
- zeros elsewhere with the same batch size as the input.
- Args:
- n : int
- The number of rows :math:`(n)`.
- B : int
- Btach size.
- device : torch.device
- Devices of the output tensor.
- dtype : torch.dtype
- Data type of the output tensor.
-
- Returns:
- The identity matrix with the shape :math:`(B, n, n)`.
- """
-
- identity = torch.eye(n, device=device, dtype=dtype)
- return identity[None].repeat(B, 1, 1)
-
-
-def normalize_homography(dst_pix_trans_src_pix, dsize_src, dsize_dst=None):
- r"""
- Normalize a given homography in pixels to [-1, 1].
- Args:
- dst_pix_trans_src_pix : torch.Tensor
- Homography/ies from source to destination to be normalized with
- shape :math:`(B, 3, 3)`.
- dsize_src : Tuple[int, int]
- Size of the source image (height, width).
- dsize_dst : Tuple[int, int]
- Size of the destination image (height, width).
-
- Returns:
- dst_norm_trans_src_norm : torch.Tensor
- The normalized homography of shape :math:`(B, 3, 3)`.
- """
- if dsize_dst is None:
- dsize_dst = dsize_src
- # source and destination sizes
- src_h, src_w = dsize_src
- dst_h, dst_w = dsize_dst
- device = dst_pix_trans_src_pix.device
- dtype = dst_pix_trans_src_pix.dtype
- # compute the transformation pixel/norm for src/dst
- src_norm_trans_src_pix = normal_transform_pixel(src_h, src_w, device, dtype).to(dst_pix_trans_src_pix)
-
- src_pix_trans_src_norm = _torch_inverse_cast(src_norm_trans_src_pix)
- dst_norm_trans_dst_pix = normal_transform_pixel(dst_h, dst_w, device, dtype).to(dst_pix_trans_src_pix)
- # compute chain transformations
- dst_norm_trans_src_norm: torch.Tensor = dst_norm_trans_dst_pix @ (dst_pix_trans_src_pix @ src_pix_trans_src_norm)
- return dst_norm_trans_src_norm
-
-
-def get_rotation_matrix2d(M, dsize):
- r"""
- Return rotation matrix for torch.affine_grid based on transformation matrix.
- Args:
- M : torch.Tensor
- Transformation matrix with shape :math:`(B, 2, 3)`.
- dsize : Tuple[int, int]
- Size of the source image (height, width).
-
- Returns:
- R : torch.Tensor
- Rotation matrix with shape :math:`(B, 2, 3)`.
- """
- H, W = dsize
- B = M.shape[0]
- center = torch.Tensor([W / 2, H / 2]).to(M.dtype).to(M.device).unsqueeze(0)
- shift_m = eye_like(3, B, M.device, M.dtype)
- shift_m[:, :2, 2] = center
-
- shift_m_inv = eye_like(3, B, M.device, M.dtype)
- shift_m_inv[:, :2, 2] = -center
-
- rotat_m = eye_like(3, B, M.device, M.dtype)
- rotat_m[:, :2, :2] = M[:, :2, :2]
- affine_m = shift_m @ rotat_m @ shift_m_inv
- return affine_m[:, :2, :] # Bx2x3
-
-
-def get_transformation_matrix(M, dsize):
- r"""
- Return transformation matrix for torch.affine_grid.
- Args:
- M : torch.Tensor
- Transformation matrix with shape :math:`(N, 2, 3)`.
- dsize : Tuple[int, int]
- Size of the source image (height, width).
-
- Returns:
- T : torch.Tensor
- Transformation matrix with shape :math:`(N, 2, 3)`.
- """
- T = get_rotation_matrix2d(M, dsize)
- T[..., 2] += M[..., 2]
- return T
-
-
-def convert_affinematrix_to_homography(A):
- r"""
- Convert to homography coordinates
- Args:
- A : torch.Tensor
- The affine matrix with shape :math:`(B,2,3)`.
-
- Returns:
- H : torch.Tensor
- The homography matrix with shape of :math:`(B,3,3)`.
- """
- H: torch.Tensor = torch.nn.functional.pad(A, [0, 0, 0, 1], "constant", value=0.0)
- H[..., -1, -1] += 1.0
- return H
-
-
-def warp_affine(src, M, dsize, mode="bilinear", padding_mode="zeros", align_corners=True):
- r"""
- Transform the src based on transformation matrix M.
- Args:
- src : torch.Tensor
- Input feature map with shape :math:`(B,C,H,W)`.
- M : torch.Tensor
- Transformation matrix with shape :math:`(B,2,3)`.
- dsize : tuple
- Tuple of output image H_out and W_out.
- mode : str
- Interpolation methods for F.grid_sample.
- padding_mode : str
- Padding methods for F.grid_sample.
- align_corners : boolean
- Parameter of F.affine_grid.
-
- Returns:
- Transformed features with shape :math:`(B,C,H,W)`.
- """
-
- B, C, H, W = src.size()
-
- # we generate a 3x3 transformation matrix from 2x3 affine
- M_3x3 = convert_affinematrix_to_homography(M)
- dst_norm_trans_src_norm = normalize_homography(M_3x3, (H, W), dsize)
-
- # src_norm_trans_dst_norm = torch.inverse(dst_norm_trans_src_norm)
- src_norm_trans_dst_norm = _torch_inverse_cast(dst_norm_trans_src_norm)
- grid = F.affine_grid(src_norm_trans_dst_norm[:, :2, :], [B, C, dsize[0], dsize[1]], align_corners=align_corners)
-
- return F.grid_sample(src.half() if grid.dtype == torch.half else src, grid, align_corners=align_corners, mode=mode, padding_mode=padding_mode)
-
-
-class Test:
- """
- Test the transformation in this file.
- The methods in this class are not supposed to be used outside of this file.
- """
-
- def __init__(self):
- pass
-
- @staticmethod
- def load_img():
- torch.manual_seed(0)
- x = torch.randn(1, 5, 16, 400, 200) * 100
- # x = torch.ones(1, 5, 16, 400, 200)
- return x
-
- @staticmethod
- def load_raw_transformation_matrix(N):
- a = 90 / 180 * np.pi
- matrix = torch.Tensor([[np.cos(a), -np.sin(a), 10], [np.sin(a), np.cos(a), 10]])
- matrix = torch.repeat_interleave(matrix.unsqueeze(0).unsqueeze(0), N, dim=1)
- return matrix
-
- @staticmethod
- def load_raw_transformation_matrix2(N, alpha):
- a = alpha / 180 * np.pi
- matrix = torch.Tensor([[np.cos(a), -np.sin(a), 0, 0], [np.sin(a), np.cos(a), 0, 0]])
- matrix = torch.repeat_interleave(matrix.unsqueeze(0).unsqueeze(0), N, dim=1)
- return matrix
-
- @staticmethod
- def test():
- img = Test.load_img()
- B, L, C, H, W = img.shape
- raw_T = Test.load_raw_transformation_matrix(5)
- T = get_transformation_matrix(raw_T.reshape(-1, 2, 3), (H, W))
- img_rot = warp_affine(img.reshape(-1, C, H, W), T, (H, W))
- print(img_rot[0, 0, :, :])
- plt.matshow(img_rot[0, 0, :, :])
- plt.show()
-
- @staticmethod
- def test_combine_roi_and_cav_mask():
- B = 2
- L = 5
- C = 16
- H = 300
- W = 400
- # 2, 5
- cav_mask = torch.Tensor([[1, 1, 1, 0, 0], [1, 0, 0, 0, 0]])
- _ = torch.zeros(B, L, C, H, W) # x
- correction_matrix = Test.load_raw_transformation_matrix2(5, 10)
- correction_matrix = torch.cat([correction_matrix, correction_matrix], dim=0)
- mask = get_roi_and_cav_mask((B, L, H, W, C), cav_mask, correction_matrix, 0.4, 4)
- plt.matshow(mask[0, :, :, 0, 0])
- plt.show()
-
-
-if __name__ == "__main__":
- os.environ["KMP_DUPLICATE_LIB_OK"] = "True"
- Test.test_combine_roi_and_cav_mask()
diff --git a/OpenCOOD/opencood/models/sub_modules/vsa.py b/OpenCOOD/opencood/models/sub_modules/vsa.py
deleted file mode 100644
index 0c120aec..00000000
--- a/OpenCOOD/opencood/models/sub_modules/vsa.py
+++ /dev/null
@@ -1,264 +0,0 @@
-import copy
-
-import torch
-import torch.nn as nn
-
-from opencood.pcdet_utils.pointnet2.pointnet2_stack import pointnet2_modules as pointnet2_stack_modules
-from opencood.pcdet_utils.pointnet2.pointnet2_stack import pointnet2_utils as pointnet2_stack_utils
-from opencood.pcdet_utils.roiaware_pool3d.roiaware_pool3d_utils import points_in_boxes_gpu
-from opencood.utils import common_utils
-
-
-def bilinear_interpolate_torch(im, x, y):
- """
- Args:
- im: (H, W, C) [y, x]
- x: (N)
- y: (N)
-
- Returns:
-
- """
- x0 = torch.floor(x).long()
- x1 = x0 + 1
-
- y0 = torch.floor(y).long()
- y1 = y0 + 1
-
- x0 = torch.clamp(x0, 0, im.shape[1] - 1)
- x1 = torch.clamp(x1, 0, im.shape[1] - 1)
- y0 = torch.clamp(y0, 0, im.shape[0] - 1)
- y1 = torch.clamp(y1, 0, im.shape[0] - 1)
-
- Ia = im[y0, x0]
- Ib = im[y1, x0]
- Ic = im[y0, x1]
- Id = im[y1, x1]
-
- wa = (x1.type_as(x) - x) * (y1.type_as(y) - y)
- wb = (x1.type_as(x) - x) * (y - y0.type_as(y))
- wc = (x - x0.type_as(x)) * (y1.type_as(y) - y)
- wd = (x - x0.type_as(x)) * (y - y0.type_as(y))
- ans = torch.t((torch.t(Ia) * wa)) + torch.t(torch.t(Ib) * wb) + torch.t(torch.t(Ic) * wc) + torch.t(torch.t(Id) * wd)
- return ans
-
-
-class VoxelSetAbstraction(nn.Module):
- def __init__(self, model_cfg, voxel_size, point_cloud_range, num_bev_features=None, num_rawpoint_features=None, **kwargs):
- super().__init__()
- self.model_cfg = model_cfg
- self.voxel_size = voxel_size
- self.point_cloud_range = point_cloud_range
-
- SA_cfg = self.model_cfg["sa_layer"]
-
- self.SA_layers = nn.ModuleList()
- self.SA_layer_names = []
- self.downsample_times_map = {}
- c_in = 0
- for src_name in self.model_cfg["features_source"]:
- if src_name in ["bev", "raw_points"]:
- continue
- self.downsample_times_map[src_name] = SA_cfg[src_name]["downsample_factor"]
- mlps = copy.copy(SA_cfg[src_name]["mlps"])
- for k in range(len(mlps)):
- mlps[k] = [mlps[k][0]] + mlps[k]
- cur_layer = pointnet2_stack_modules.StackSAModuleMSG(
- radii=SA_cfg[src_name]["pool_radius"],
- nsamples=SA_cfg[src_name]["n_sample"],
- mlps=mlps,
- use_xyz=True,
- pool_method="max_pool",
- )
- self.SA_layers.append(cur_layer)
- self.SA_layer_names.append(src_name)
-
- c_in += sum([x[-1] for x in mlps])
-
- if "bev" in self.model_cfg["features_source"]:
- c_bev = num_bev_features
- c_in += c_bev
-
- if "raw_points" in self.model_cfg["features_source"]:
- mlps = copy.copy(SA_cfg["raw_points"]["mlps"])
- for k in range(len(mlps)):
- mlps[k] = [num_rawpoint_features - 3] + mlps[k]
-
- self.SA_rawpoints = pointnet2_stack_modules.StackSAModuleMSG(
- radii=SA_cfg["raw_points"]["pool_radius"], nsamples=SA_cfg["raw_points"]["n_sample"], mlps=mlps, use_xyz=True, pool_method="max_pool"
- )
- c_in += sum([x[-1] for x in mlps])
-
- self.vsa_point_feature_fusion = nn.Sequential(
- nn.Linear(c_in, self.model_cfg["num_out_features"], bias=False),
- nn.BatchNorm1d(self.model_cfg["num_out_features"]),
- nn.ReLU(),
- )
- self.num_point_features = self.model_cfg["num_out_features"]
- self.num_point_features_before_fusion = c_in # noqa: DC05
-
- def interpolate_from_bev_features(self, keypoints, bev_features, batch_size, bev_stride):
- x_idxs = (keypoints[:, :, 0] - self.point_cloud_range[0]) / self.voxel_size[0]
- y_idxs = (keypoints[:, :, 1] - self.point_cloud_range[1]) / self.voxel_size[1]
- x_idxs = x_idxs / bev_stride
- y_idxs = y_idxs / bev_stride
-
- point_bev_features_list = []
- for k in range(batch_size):
- cur_x_idxs = x_idxs[k]
- cur_y_idxs = y_idxs[k]
- cur_bev_features = bev_features[k].permute(1, 2, 0) # (H, W, C)
- point_bev_features = bilinear_interpolate_torch(cur_bev_features, cur_x_idxs, cur_y_idxs)
- point_bev_features_list.append(point_bev_features.unsqueeze(dim=0))
-
- point_bev_features = torch.cat(point_bev_features_list, dim=0) # (B, N, C0)
- return point_bev_features
-
- def get_sampled_points(self, batch_dict):
- batch_size = batch_dict["batch_size"]
- if self.model_cfg["point_source"] == "raw_points":
- src_points = batch_dict["origin_lidar"][:, 1:]
- batch_indices = batch_dict["origin_lidar"][:, 0].long()
- elif self.model_cfg["point_source"] == "voxel_centers":
- src_points = common_utils.get_voxel_centers(
- batch_dict["voxel_coords"][:, 1:4], downsample_times=1, voxel_size=self.voxel_size, point_cloud_range=self.point_cloud_range
- )
- batch_indices = batch_dict["voxel_coords"][:, 0].long()
- else:
- raise NotImplementedError
-
- keypoints_batch = torch.randn((batch_size, self.model_cfg["num_keypoints"], 4), device=src_points.device)
- keypoints_batch[..., 0] = keypoints_batch[..., 0] * 140
- keypoints_batch[..., 1] = keypoints_batch[..., 0] * 40
- # points with height flag 10 are padding/invalid, for later filtering
- keypoints_batch[..., 2] = 10.0
- for bs_idx in range(batch_size):
- bs_mask = batch_indices == bs_idx
- sampled_points = src_points[bs_mask].unsqueeze(dim=0) # (1, N, 3)
- # sample points with FPS
- # some cropped pcd may have very few points, select various number
- # of points to ensure similar sample density
- # 50000 is approximately the number of points in one full pcd
- num_kpts = int(self.model_cfg["num_keypoints"] * sampled_points.shape[1] / 50000) + 1
- num_kpts = min(num_kpts, self.model_cfg["num_keypoints"])
- cur_pt_idxs = pointnet2_stack_utils.furthest_point_sample(sampled_points[:, :, 0:3].contiguous(), num_kpts).long()
-
- if sampled_points.shape[1] < num_kpts:
- empty_num = num_kpts - sampled_points.shape[1]
- cur_pt_idxs[0, -empty_num:] = cur_pt_idxs[0, :empty_num]
-
- keypoints = sampled_points[0][cur_pt_idxs[0]].unsqueeze(dim=0)
-
- keypoints_batch[bs_idx, : len(keypoints[0]), :] = keypoints
-
- # keypoints = torch.cat(keypoints_list, dim=0) # (B, M, 3)
- return keypoints_batch
-
- def forward(self, batch_dict):
- """
- Args:
- batch_dict:
- batch_size:
- keypoints: (B, num_keypoints, 3)
- multi_scale_3d_features: {
- 'x_conv4': ...
- }
- points: optional (N, 1 + 3 + C) [bs_idx, x, y, z, ...]
- spatial_features: optional
- spatial_features_stride: optional
-
- Returns:
- point_features: (N, C)
- point_coords: (N, 4)
-
- """
- keypoints = self.get_sampled_points(batch_dict) # BxNx4
- kpt_mask1 = torch.logical_and(keypoints[..., 2] > -2.8, keypoints[..., 2] < 1.0)
- kpt_mask2 = None
- # Only select the points that are in the predicted bounding boxes
- if "det_boxes" in batch_dict:
- dets_list = batch_dict["det_boxes"]
- max_len = max([len(dets) for dets in dets_list])
- boxes = torch.zeros((len(dets_list), max_len, 7), dtype=dets_list[0].dtype, device=dets_list[0].device)
- for i, dets in enumerate(dets_list):
- if len(dets) == 0:
- continue
- cur_dets = dets.clone()
- if self.model_cfg["enlarge_selection_boxes"]:
- cur_dets[:, 3:6] += 0.5
- boxes[i, : len(dets)] = cur_dets
- # mask out some keypoints to spare the GPU storage
- kpt_mask2 = points_in_boxes_gpu(keypoints[..., :3], boxes) >= 0
-
- kpt_mask = torch.logical_and(kpt_mask1, kpt_mask2) if kpt_mask2 is not None else kpt_mask1
- # Ensure there are more than 2 points are selected to satisfy the
- # condition of batch norm in the FC layers of feature fusion module
- if (kpt_mask).sum() < 2:
- kpt_mask[0, torch.randint(0, 1024, (2,))] = True
-
- point_features_list = []
- if "bev" in self.model_cfg["features_source"]:
- point_bev_features = self.interpolate_from_bev_features(
- keypoints[..., :3], batch_dict["spatial_features"], batch_dict["batch_size"], bev_stride=batch_dict["spatial_features_stride"]
- )
- point_features_list.append(point_bev_features[kpt_mask])
-
- batch_size, _, _ = keypoints.shape
-
- new_xyz = keypoints[kpt_mask]
- new_xyz_batch_cnt = torch.tensor([(mask).sum() for mask in kpt_mask], device=new_xyz.device).int()
-
- if "raw_points" in self.model_cfg["features_source"]:
- raw_points = batch_dict["origin_lidar"]
- xyz = raw_points[:, 1:4]
- xyz_batch_cnt = xyz.new_zeros(batch_size).int()
- indices = raw_points[:, 0].long()
- for bs_idx in range(batch_size):
- xyz_batch_cnt[bs_idx] = (indices == bs_idx).sum()
- point_features = None
-
- _, pooled_features = self.SA_rawpoints(
- xyz=xyz.contiguous(),
- xyz_batch_cnt=xyz_batch_cnt,
- new_xyz=new_xyz[:, :3].contiguous(),
- new_xyz_batch_cnt=new_xyz_batch_cnt,
- features=point_features,
- )
- point_features_list.append(pooled_features)
-
- for k, src_name in enumerate(self.SA_layer_names):
- cur_coords = batch_dict["multi_scale_3d_features"][src_name].indices
- xyz = common_utils.get_voxel_centers(
- cur_coords[:, 1:4],
- downsample_times=self.downsample_times_map[src_name],
- voxel_size=self.voxel_size,
- point_cloud_range=self.point_cloud_range,
- )
- xyz_batch_cnt = xyz.new_zeros(batch_size).int()
- for bs_idx in range(batch_size):
- xyz_batch_cnt[bs_idx] = (cur_coords[:, 0] == bs_idx).sum()
-
- _, pooled_features = self.SA_layers[k](
- xyz=xyz.contiguous(),
- xyz_batch_cnt=xyz_batch_cnt,
- new_xyz=new_xyz[:, :3].contiguous(),
- new_xyz_batch_cnt=new_xyz_batch_cnt,
- features=batch_dict["multi_scale_3d_features"][src_name].features.contiguous(),
- )
-
- point_features_list.append(pooled_features)
-
- point_features = torch.cat(point_features_list, dim=1)
-
- batch_dict["point_features_before_fusion"] = point_features.view(-1, point_features.shape[-1])
- point_features = self.vsa_point_feature_fusion(point_features.view(-1, point_features.shape[-1]))
-
- cur_idx = 0
- batch_dict["point_features"] = []
- batch_dict["point_coords"] = []
- for num in new_xyz_batch_cnt:
- batch_dict["point_features"].append(point_features[cur_idx : cur_idx + num])
- batch_dict["point_coords"].append(new_xyz[cur_idx : cur_idx + num])
- cur_idx += num
-
- return batch_dict
diff --git a/OpenCOOD/opencood/models/voxel_net.py b/OpenCOOD/opencood/models/voxel_net.py
deleted file mode 100644
index 0d7799f3..00000000
--- a/OpenCOOD/opencood/models/voxel_net.py
+++ /dev/null
@@ -1,231 +0,0 @@
-import torch.nn as nn
-import torch.nn.functional as F
-import torch
-
-from opencood.models.sub_modules.pillar_vfe import PillarVFE
-
-
-# conv2d + bn + relu
-class Conv2d(nn.Module):
- def __init__(self, in_channels, out_channels, k, s, p, activation=True, batch_norm=True):
- super(Conv2d, self).__init__()
- self.conv = nn.Conv2d(in_channels, out_channels, kernel_size=k, stride=s, padding=p)
- if batch_norm:
- self.bn = nn.BatchNorm2d(out_channels)
- else:
- self.bn = None
- self.activation = activation
-
- def forward(self, x):
- x = self.conv(x)
- if self.bn is not None:
- x = self.bn(x)
- if self.activation:
- return F.relu(x, inplace=True)
- else:
- return x
-
-
-# conv3d + bn + relu
-class Conv3d(nn.Module):
- def __init__(self, in_channels, out_channels, k, s, p, batch_norm=True):
- super(Conv3d, self).__init__()
- self.conv = nn.Conv3d(in_channels, out_channels, kernel_size=k, stride=s, padding=p)
- if batch_norm:
- self.bn = nn.BatchNorm3d(out_channels)
- else:
- self.bn = None
-
- def forward(self, x):
- x = self.conv(x)
- if self.bn is not None:
- x = self.bn(x)
-
- return F.relu(x, inplace=True)
-
-
-# Fully Connected Network
-class FCN(nn.Module):
- def __init__(self, cin, cout):
- super(FCN, self).__init__()
- self.cout = cout
- self.linear = nn.Linear(cin, cout)
- self.bn = nn.BatchNorm1d(cout)
-
- def forward(self, x):
- # KK is the stacked k across batch
- kk, t, _ = x.shape
- x = self.linear(x.view(kk * t, -1))
- x = F.relu(self.bn(x))
- return x.view(kk, t, -1)
-
-
-# Voxel Feature Encoding layer
-class VFE(nn.Module):
- def __init__(self, cin, cout, T):
- super(VFE, self).__init__()
- assert cout % 2 == 0
- self.units = cout // 2
- self.fcn = FCN(cin, self.units)
- self.T = T
-
- def forward(self, x, mask):
- # point-wise feature
- pwf = self.fcn(x)
- # locally aggregated feature
- laf = torch.max(pwf, 1)[0].unsqueeze(1).repeat(1, self.T, 1)
- # point-wise concat feature
- pwcf = torch.cat((pwf, laf), dim=2)
- # apply mask
- mask = mask.unsqueeze(2).repeat(1, 1, self.units * 2)
- pwcf = pwcf * mask.float()
-
- return pwcf
-
-
-# Stacked Voxel Feature Encoding
-class SVFE(nn.Module):
- def __init__(self, T):
- super(SVFE, self).__init__()
- self.vfe_1 = VFE(7, 32, T)
- self.vfe_2 = VFE(32, 128, T)
- self.fcn = FCN(128, 128)
-
- def forward(self, x):
- mask = torch.ne(torch.max(x, 2)[0], 0)
- x = self.vfe_1(x, mask)
- x = self.vfe_2(x, mask)
- x = self.fcn(x)
- # element-wise max pooling
- x = torch.max(x, 1)[0]
- return x
-
-
-# Convolutional Middle Layer
-class CML(nn.Module):
- def __init__(self):
- super(CML, self).__init__()
- self.conv3d_1 = Conv3d(64, 64, 3, s=(2, 1, 1), p=(1, 1, 1))
- self.conv3d_2 = Conv3d(64, 64, 3, s=(1, 1, 1), p=(0, 1, 1))
- self.conv3d_3 = Conv3d(64, 64, 3, s=(2, 1, 1), p=(1, 1, 1))
-
- def forward(self, x):
- x = self.conv3d_1(x)
- x = self.conv3d_2(x)
- x = self.conv3d_3(x)
- return x
-
-
-# Region Proposal Network
-class RPN(nn.Module):
- def __init__(self, anchor_num=2):
- super(RPN, self).__init__()
- self.anchor_num = anchor_num
-
- self.block_1 = [Conv2d(128, 128, 3, 2, 1)]
- self.block_1 += [Conv2d(128, 128, 3, 1, 1) for _ in range(3)]
- self.block_1 = nn.Sequential(*self.block_1)
-
- self.block_2 = [Conv2d(128, 128, 3, 2, 1)]
- self.block_2 += [Conv2d(128, 128, 3, 1, 1) for _ in range(5)]
- self.block_2 = nn.Sequential(*self.block_2)
-
- self.block_3 = [Conv2d(128, 256, 3, 2, 1)]
- self.block_3 += [nn.Conv2d(256, 256, 3, 1, 1) for _ in range(5)]
- self.block_3 = nn.Sequential(*self.block_3)
-
- self.deconv_1 = nn.Sequential(nn.ConvTranspose2d(256, 256, 4, 4, 0), nn.BatchNorm2d(256))
- self.deconv_2 = nn.Sequential(nn.ConvTranspose2d(128, 256, 2, 2, 0), nn.BatchNorm2d(256))
- self.deconv_3 = nn.Sequential(nn.ConvTranspose2d(128, 256, 1, 1, 0), nn.BatchNorm2d(256))
-
- self.score_head = Conv2d(768, self.anchor_num, 1, 1, 0, activation=False, batch_norm=False)
- self.reg_head = Conv2d(768, 7 * self.anchor_num, 1, 1, 0, activation=False, batch_norm=False)
-
- def forward(self, x):
- x = self.block_1(x)
- x_skip_1 = x
- x = self.block_2(x)
- x_skip_2 = x
- x = self.block_3(x)
- x_0 = self.deconv_1(x)
- x_1 = self.deconv_2(x_skip_2)
- x_2 = self.deconv_3(x_skip_1)
- x = torch.cat((x_0, x_1, x_2), 1)
- return self.score_head(x), self.reg_head(x)
-
-
-class VoxelNet(nn.Module):
- def __init__(self, args):
- super(VoxelNet, self).__init__()
- self.svfe = PillarVFE(args["pillar_vfe"], num_point_features=4, voxel_size=args["voxel_size"], point_cloud_range=args["lidar_range"])
-
- # self.svfe = SVFE(args['T'])
- self.cml = CML()
- self.rpn = RPN(args["anchor_num"])
-
- self.N = args["N"]
- self.D = args["D"]
- self.H = args["H"]
- self.W = args["W"]
- self.T = args["T"]
- self.anchor_num = args["anchor_num"]
-
- def voxel_indexing(self, sparse_features, coords, batch_size):
- """
- Scatter sparse voxel features into a dense model batch.
-
- Parameters
- ----------
- sparse_features : torch.Tensor
- Encoded features for non-empty voxels.
- coords : torch.Tensor
- Batched voxel coordinates.
- batch_size : int
- Number of point clouds represented by the coordinates.
-
- Returns
- -------
- torch.Tensor
- Dense voxel feature tensor.
- """
- dim = sparse_features.shape[-1]
- dense_feature = torch.zeros(
- dim,
- batch_size,
- self.D,
- self.H,
- self.W,
- device=sparse_features.device,
- dtype=sparse_features.dtype,
- )
- coords = coords.long()
-
- dense_feature[:, coords[:, 0], coords[:, 1], coords[:, 2], coords[:, 3]] = sparse_features.transpose(0, 1)
-
- return dense_feature.transpose(0, 1)
-
- def forward(self, data_dict):
- voxel_features = data_dict["processed_lidar"]["voxel_features"]
- voxel_coords = data_dict["processed_lidar"]["voxel_coords"]
- voxel_num_points = data_dict["processed_lidar"]["voxel_num_points"]
- batch_size = data_dict.get("batch_size", self.N)
-
- batch_dict = {"voxel_features": voxel_features, "voxel_coords": voxel_coords, "voxel_num_points": voxel_num_points}
-
- # feature learning network
- vwfs = self.svfe(batch_dict)["pillar_features"]
-
- vwfs = self.voxel_indexing(vwfs, voxel_coords, batch_size)
-
- # convolutional middle network
- vwfs = self.cml(vwfs)
-
- # region proposal network
-
- # merge the depth and feature dim into one, output probability score
- # map and regression map
- psm, rm = self.rpn(vwfs.view(batch_size, -1, self.H, self.W))
-
- output_dict = {"psm": psm, "rm": rm}
-
- return output_dict
diff --git a/OpenCOOD/opencood/models/voxel_net_intermediate.py b/OpenCOOD/opencood/models/voxel_net_intermediate.py
deleted file mode 100644
index 97474b50..00000000
--- a/OpenCOOD/opencood/models/voxel_net_intermediate.py
+++ /dev/null
@@ -1,246 +0,0 @@
-"""
-VoxelNet for intermediate fusion
-"""
-
-import torch.nn as nn
-import torch.nn.functional as F
-import torch
-import numpy as np
-
-from opencood.models.communication_adapters.intermediate import SpatialFeatureCommunicationAdapter
-from opencood.models.voxel_net import RPN, CML
-from opencood.models.sub_modules.pillar_vfe import PillarVFE
-from opencood.models.fuse_modules.self_attn import AttFusion
-from opencood.models.sub_modules.auto_encoder import AutoEncoder
-
-
-# conv2d + bn + relu
-class Conv2d(nn.Module):
- def __init__(self, in_channels, out_channels, k, s, p, activation=True, batch_norm=True, bias=True):
- super(Conv2d, self).__init__()
- self.conv = nn.Conv2d(in_channels, out_channels, kernel_size=k, stride=s, padding=p, bias=bias)
- if batch_norm:
- self.bn = nn.BatchNorm2d(out_channels)
- else:
- self.bn = None
- self.activation = activation
-
- def forward(self, x):
- x = self.conv(x)
- if self.bn is not None:
- x = self.bn(x)
- if self.activation:
- return F.relu(x, inplace=True)
- else:
- return x
-
-
-class NaiveFusion(nn.Module):
- def __init__(self):
- super(NaiveFusion, self).__init__()
- self.conv1 = Conv2d(128 * 5, 256, 3, 1, 1, batch_norm=False, bias=False)
- self.conv2 = Conv2d(256, 128, 3, 1, 1)
-
- def forward(self, x):
- x = self.conv1(x)
- x = self.conv2(x)
-
- return x
-
-
-class VoxelNetIntermediate(nn.Module):
- communication_adapter_class = SpatialFeatureCommunicationAdapter
-
- def __init__(self, args):
- super(VoxelNetIntermediate, self).__init__()
- self.svfe = PillarVFE(args["pillar_vfe"], num_point_features=4, voxel_size=args["voxel_size"], point_cloud_range=args["lidar_range"])
- self.cml = CML()
- self.fusion_net = AttFusion(128)
- self.rpn = RPN(args["anchor_num"])
-
- self.N = args["N"]
- self.D = args["D"]
- self.H = args["H"]
- self.W = args["W"]
- self.T = args["T"]
- self.anchor_num = args["anchor_num"]
-
- self.compression = False
- if "compression" in args and args["compression"] > 0:
- self.compression = True
- self.compression_layer = AutoEncoder(128, args["compression"])
-
- def voxel_indexing(self, sparse_features, coords, batch_size):
- """
- Scatter sparse VoxelNet features into a dense 3D tensor.
-
- Parameters
- ----------
- sparse_features : torch.Tensor
- Learned feature vector for every non-empty voxel.
- coords : torch.Tensor
- Batch, depth, height, and width index for every sparse feature.
- batch_size : int
- Number of agents represented by the sparse input.
-
- Returns
- -------
- torch.Tensor
- Dense tensor shaped ``(batch_size, channels, D, H, W)``.
- """
- dim = sparse_features.shape[-1]
-
- dense_feature = torch.zeros(
- dim,
- batch_size,
- self.D,
- self.H,
- self.W,
- device=sparse_features.device,
- dtype=sparse_features.dtype,
- )
- coords = coords.long()
-
- dense_feature[:, coords[:, 0], coords[:, 1], coords[:, 2], coords[:, 3]] = sparse_features.transpose(0, 1)
-
- return dense_feature.transpose(0, 1)
-
- def regroup(self, dense_feature, record_len):
- """
- Regroup the data based on the record_len.
-
- Parameters
- ----------
- dense_feature : torch.Tensor
- N, C, H, W
- record_len : list
- [sample1_len, sample2_len, ...]
-
- Returns
- -------
- regroup_feature : torch.Tensor
- B, 5C, H, W
- """
- cum_sum_len = list(np.cumsum(record_len))
- split_features = torch.tensor_split(dense_feature, cum_sum_len[:-1])
- regroup_features = []
-
- for split_feature in split_features:
- # M, C, H, W
- feature_shape = split_feature.shape
-
- # the maximum M is 5 as most 5 cavs
- padding_len = 5 - feature_shape[0]
- padding_tensor = torch.zeros(padding_len, feature_shape[1], feature_shape[2], feature_shape[3])
- padding_tensor = padding_tensor.to(split_feature.device)
-
- split_feature = torch.cat([split_feature, padding_tensor], dim=0)
-
- # 1, 5C, H, W
- split_feature = split_feature.view(-1, feature_shape[2], feature_shape[3]).unsqueeze(0)
- regroup_features.append(split_feature)
-
- # B, 5C, H, W
- regroup_features = torch.cat(regroup_features, dim=0)
-
- return regroup_features
-
- def forward(self, data_dict):
- if "intermediate_features" in data_dict:
- return self.fuse_agents(data_dict)
-
- voxel_features = data_dict["processed_lidar"]["voxel_features"]
- voxel_coords = data_dict["processed_lidar"]["voxel_coords"]
- voxel_num_points = data_dict["processed_lidar"]["voxel_num_points"]
- record_len = data_dict["record_len"]
-
- batch_dict = {"voxel_features": voxel_features, "voxel_coords": voxel_coords, "voxel_num_points": voxel_num_points}
-
- record_len_tmp = record_len.cpu() if voxel_coords.is_cuda else record_len
-
- record_len_tmp = list(record_len_tmp.numpy())
-
- self.N = sum(record_len_tmp)
-
- # feature learning network
- vwfs = self.svfe(batch_dict)["pillar_features"]
-
- vwfs = self.voxel_indexing(vwfs, voxel_coords, self.N)
-
- # convolutional middle network
- vwfs = self.cml(vwfs)
- # convert from 3d to 2d N C H W
- vmfs = vwfs.view(self.N, -1, self.H, self.W)
-
- # compression layer
- if self.compression:
- vmfs = self.compression_layer(vmfs)
-
- # information naive fusion
- vmfs_fusion = self.fusion_net(vmfs, record_len)
-
- # region proposal network
- # merge the depth and feature dim into one, output probability score
- # map and regression map
- psm, rm = self.rpn(vmfs_fusion)
-
- output_dict = {"psm": psm, "rm": rm}
-
- return output_dict
-
- def encode_agent(self, data_dict):
- """
- Encode one agent through VoxelNet's convolutional middle layers.
-
- Parameters
- ----------
- data_dict : dict
- Sender-local preprocessed voxel input.
-
- Returns
- -------
- dict
- Learned spatial feature map before attention fusion.
- """
- processed_lidar = data_dict["processed_lidar"]
- batch_dict = {
- "voxel_features": processed_lidar["voxel_features"],
- "voxel_coords": processed_lidar["voxel_coords"],
- "voxel_num_points": processed_lidar["voxel_num_points"],
- }
- voxel_features = self.svfe(batch_dict)["pillar_features"]
- dense_features = self.voxel_indexing(
- voxel_features,
- processed_lidar["voxel_coords"],
- data_dict.get("batch_size", 1),
- )
- spatial_features = self.cml(dense_features).view(
- data_dict.get("batch_size", 1),
- -1,
- self.H,
- self.W,
- )
- if self.compression:
- spatial_features = self.compression_layer.encode(spatial_features)
- return {"spatial_features": spatial_features}
-
- def fuse_agents(self, data_dict):
- """
- Fuse received VoxelNet features and run its proposal network.
-
- Parameters
- ----------
- data_dict : dict
- Receiver input containing learned features and ``record_len``.
-
- Returns
- -------
- dict
- Classification and regression maps.
- """
- spatial_features = data_dict["intermediate_features"]["spatial_features"]
- if self.compression:
- spatial_features = self.compression_layer.decode(spatial_features)
- fused_features = self.fusion_net(spatial_features, data_dict["record_len"])
- psm, rm = self.rpn(fused_features)
- return {"psm": psm, "rm": rm}
diff --git a/OpenCOOD/opencood/pcdet_utils/CMakeLists.txt b/OpenCOOD/opencood/pcdet_utils/CMakeLists.txt
deleted file mode 100644
index 1ce46480..00000000
--- a/OpenCOOD/opencood/pcdet_utils/CMakeLists.txt
+++ /dev/null
@@ -1,83 +0,0 @@
-separate_arguments(OPENCDA_TORCH_CXX_FLAGS NATIVE_COMMAND "${TORCH_CXX_FLAGS}")
-
-function(opencda_add_cuda_extension target package_directory)
- Python3_add_library(${target} MODULE WITH_SOABI ${ARGN})
-
- target_include_directories(${target} PRIVATE ${TORCH_INCLUDE_DIRS})
- target_link_libraries(${target} PRIVATE
- ${TORCH_LIBRARIES}
- "${OPENCDA_TORCH_PYTHON_LIBRARY}"
- CUDA::cudart
- )
- target_compile_definitions(${target} PRIVATE
- TORCH_API_INCLUDE_EXTENSION_H
- TORCH_EXTENSION_NAME=${target}
- )
- target_compile_options(${target} PRIVATE
- ${OPENCDA_TORCH_CXX_FLAGS}
- $<$:-D__CUDA_NO_HALF_OPERATORS__>
- $<$:-D__CUDA_NO_HALF_CONVERSIONS__>
- $<$:-D__CUDA_NO_BFLOAT16_CONVERSIONS__>
- $<$:-D__CUDA_NO_HALF2_OPERATORS__>
- $<$:--expt-relaxed-constexpr>
- )
-
- set_target_properties(${target} PROPERTIES
- CUDA_ARCHITECTURES OFF
- INSTALL_RPATH "${OPENCDA_TORCH_RUNTIME_LIBRARY_DIR}"
- )
-
- install(
- TARGETS ${target}
- LIBRARY DESTINATION "OpenCOOD/opencood/pcdet_utils/${package_directory}"
- )
-endfunction()
-
-opencda_add_cuda_extension(
- iou3d_nms_cuda
- iou3d_nms
- iou3d_nms/src/iou3d_cpu.cpp
- iou3d_nms/src/iou3d_nms_api.cpp
- iou3d_nms/src/iou3d_nms.cpp
- iou3d_nms/src/iou3d_nms_kernel.cu
-)
-
-opencda_add_cuda_extension(
- roiaware_pool3d_cuda
- roiaware_pool3d
- roiaware_pool3d/src/roiaware_pool3d.cpp
- roiaware_pool3d/src/roiaware_pool3d_kernel.cu
-)
-
-opencda_add_cuda_extension(
- pointnet2_stack_cuda
- pointnet2/pointnet2_stack
- pointnet2/pointnet2_stack/src/pointnet2_api.cpp
- pointnet2/pointnet2_stack/src/ball_query.cpp
- pointnet2/pointnet2_stack/src/ball_query_gpu.cu
- pointnet2/pointnet2_stack/src/group_points.cpp
- pointnet2/pointnet2_stack/src/group_points_gpu.cu
- pointnet2/pointnet2_stack/src/sampling.cpp
- pointnet2/pointnet2_stack/src/sampling_gpu.cu
- pointnet2/pointnet2_stack/src/interpolate.cpp
- pointnet2/pointnet2_stack/src/interpolate_gpu.cu
-)
-
-opencda_add_cuda_extension(
- pointnet2_batch_cuda
- pointnet2/pointnet2_batch
- pointnet2/pointnet2_batch/src/pointnet2_api.cpp
- pointnet2/pointnet2_batch/src/ball_query.cpp
- pointnet2/pointnet2_batch/src/ball_query_gpu.cu
- pointnet2/pointnet2_batch/src/group_points.cpp
- pointnet2/pointnet2_batch/src/group_points_gpu.cu
- pointnet2/pointnet2_batch/src/interpolate.cpp
- pointnet2/pointnet2_batch/src/interpolate_gpu.cu
- pointnet2/pointnet2_batch/src/sampling.cpp
- pointnet2/pointnet2_batch/src/sampling_gpu.cu
-)
-
-install(
- SCRIPT
- "${CMAKE_CURRENT_LIST_DIR}/cmake/WriteCudaArtifactManifest.cmake"
-)
diff --git a/OpenCOOD/opencood/pcdet_utils/__init__.py b/OpenCOOD/opencood/pcdet_utils/__init__.py
deleted file mode 100644
index bcbe18dd..00000000
--- a/OpenCOOD/opencood/pcdet_utils/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-"""This is used to save the utility functions for OpenPCDet"""
diff --git a/OpenCOOD/opencood/pcdet_utils/cmake/WriteCudaArtifactManifest.cmake b/OpenCOOD/opencood/pcdet_utils/cmake/WriteCudaArtifactManifest.cmake
deleted file mode 100644
index 4b6a2f7a..00000000
--- a/OpenCOOD/opencood/pcdet_utils/cmake/WriteCudaArtifactManifest.cmake
+++ /dev/null
@@ -1,17 +0,0 @@
-file(
- GLOB_RECURSE installed_cuda_artifacts
- RELATIVE "${CMAKE_INSTALL_PREFIX}"
- "${CMAKE_INSTALL_PREFIX}/OpenCOOD/opencood/pcdet_utils/*_cuda*.so"
-)
-list(SORT installed_cuda_artifacts)
-
-string(REPLACE ";" "\n" cuda_manifest "${installed_cuda_artifacts}")
-if(NOT cuda_manifest STREQUAL "")
- string(APPEND cuda_manifest "\n")
-endif()
-
-file(
- WRITE
- "${CMAKE_INSTALL_PREFIX}/cuda-artifacts.manifest"
- "${cuda_manifest}"
-)
diff --git a/OpenCOOD/opencood/pcdet_utils/iou3d_nms/iou3d_nms_utils.py b/OpenCOOD/opencood/pcdet_utils/iou3d_nms/iou3d_nms_utils.py
deleted file mode 100644
index 1bac606d..00000000
--- a/OpenCOOD/opencood/pcdet_utils/iou3d_nms/iou3d_nms_utils.py
+++ /dev/null
@@ -1,121 +0,0 @@
-"""
-3D IoU Calculation and Rotated NMS
-Written by Shaoshuai Shi
-All Rights Reserved 2019-2020.
-"""
-
-import torch
-
-from opencood.pcdet_utils.iou3d_nms import iou3d_nms_cuda
-
-
-def aligned_boxes_iou3d_gpu(boxes_a, boxes_b, return_union=False):
- """
- Args:
- boxes_a: (N, 7) [x, y, z, dx, dy, dz, heading]
- boxes_b: (N, 7) [x, y, z, dx, dy, dz, heading]
-
- Returns:
- ans_iou: (N, 1)
- """
- assert boxes_a.shape[1] == boxes_b.shape[1] == 7
- assert boxes_a.shape[0] == boxes_b.shape[0]
- # height overlap
- boxes_a_height_max = (boxes_a[:, 2] + boxes_a[:, 5] / 2).view(-1, 1)
- boxes_a_height_min = (boxes_a[:, 2] - boxes_a[:, 5] / 2).view(-1, 1)
- boxes_b_height_max = (boxes_b[:, 2] + boxes_b[:, 5] / 2).view(-1, 1)
- boxes_b_height_min = (boxes_b[:, 2] - boxes_b[:, 5] / 2).view(-1, 1)
-
- # bev overlap
- overlaps_bev = torch.cuda.FloatTensor(torch.Size((boxes_a.shape[0], boxes_b.shape[0]))).zero_() # (N, M)
- iou3d_nms_cuda.boxes_overlap_bev_gpu(boxes_a.contiguous(), boxes_b.contiguous(), overlaps_bev)
- overlaps_bev = torch.diagonal(overlaps_bev).reshape(-1, 1)
-
- max_of_min = torch.max(boxes_a_height_min, boxes_b_height_min)
- min_of_max = torch.min(boxes_a_height_max, boxes_b_height_max)
- overlaps_h = torch.clamp(min_of_max - max_of_min, min=0)
-
- # 3d iou
- overlaps_3d = overlaps_bev * overlaps_h
-
- vol_a = (boxes_a[:, 3] * boxes_a[:, 4] * boxes_a[:, 5]).view(-1, 1)
- vol_b = (boxes_b[:, 3] * boxes_b[:, 4] * boxes_b[:, 5]).view(-1, 1)
- union = torch.clamp(vol_a + vol_b - overlaps_3d, min=1e-6)
- iou3d = overlaps_3d / union
- if return_union:
- return iou3d, union
- return iou3d
-
-
-def boxes_iou3d_gpu(boxes_a, boxes_b, return_union=False):
- """
- Args:
- boxes_a: (N, 7) [x, y, z, dx, dy, dz, heading]
- boxes_b: (N, 7) [x, y, z, dx, dy, dz, heading]
-
- Returns:
- ans_iou: (N, M)
- """
- assert boxes_a.shape[1] == boxes_b.shape[1] == 7
-
- # height overlap
- boxes_a_height_max = (boxes_a[:, 2] + boxes_a[:, 5] / 2).view(-1, 1)
- boxes_a_height_min = (boxes_a[:, 2] - boxes_a[:, 5] / 2).view(-1, 1)
- boxes_b_height_max = (boxes_b[:, 2] + boxes_b[:, 5] / 2).view(1, -1)
- boxes_b_height_min = (boxes_b[:, 2] - boxes_b[:, 5] / 2).view(1, -1)
-
- # bev overlap
- overlaps_bev = torch.cuda.FloatTensor(torch.Size((boxes_a.shape[0], boxes_b.shape[0]))).zero_() # (N, M)
- iou3d_nms_cuda.boxes_overlap_bev_gpu(boxes_a.contiguous(), boxes_b.contiguous(), overlaps_bev)
-
- max_of_min = torch.max(boxes_a_height_min, boxes_b_height_min)
- min_of_max = torch.min(boxes_a_height_max, boxes_b_height_max)
- overlaps_h = torch.clamp(min_of_max - max_of_min, min=0)
-
- # 3d iou
- overlaps_3d = overlaps_bev * overlaps_h
-
- vol_a = (boxes_a[:, 3] * boxes_a[:, 4] * boxes_a[:, 5]).view(-1, 1)
- vol_b = (boxes_b[:, 3] * boxes_b[:, 4] * boxes_b[:, 5]).view(1, -1)
- union = torch.clamp(vol_a + vol_b - overlaps_3d, min=1e-6)
- iou3d = overlaps_3d / union
- if return_union:
- return iou3d, union
- return iou3d
-
-
-def nms_gpu(boxes, scores, thresh, pre_maxsize=None, **kwargs):
- """
- Operate on rotated bev boxes[x,y,dx,dy,heading]
- :param boxes: (N, 7) [x, y, z, dx, dy, dz, heading]
- :param scores: (N)
- :param thresh:
- :return:
- """
- assert boxes.shape[1] == 7
- order = scores.sort(0, descending=True)[1]
- if pre_maxsize is not None:
- order = order[:pre_maxsize]
-
- boxes = boxes[order].contiguous()
- keep = torch.LongTensor(boxes.size(0))
- num_out = iou3d_nms_cuda.nms_gpu(boxes, keep, thresh)
- return order[keep[:num_out].cuda()].contiguous(), None
-
-
-def nms_normal_gpu(boxes, scores, thresh, **kwargs):
- """
- Ignore heading and operate on bev boxes[x,y,dx,dy]
- :param boxes: (N, 7) [x, y, z, dx, dy, dz, heading]
- :param scores: (N)
- :param thresh:
- :return:
- """
- assert boxes.shape[1] == 7
- order = scores.sort(0, descending=True)[1]
-
- boxes = boxes[order].contiguous()
-
- keep = torch.LongTensor(boxes.size(0))
- num_out = iou3d_nms_cuda.nms_normal_gpu(boxes, keep, thresh)
- return order[keep[:num_out].cuda()].contiguous(), None
diff --git a/OpenCOOD/opencood/pcdet_utils/iou3d_nms/src/iou3d_cpu.cpp b/OpenCOOD/opencood/pcdet_utils/iou3d_nms/src/iou3d_cpu.cpp
deleted file mode 100644
index 22ab25e0..00000000
--- a/OpenCOOD/opencood/pcdet_utils/iou3d_nms/src/iou3d_cpu.cpp
+++ /dev/null
@@ -1,252 +0,0 @@
-/*
-3D Rotated IoU Calculation (CPU)
-Written by Shaoshuai Shi
-All Rights Reserved 2020.
-*/
-
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include "iou3d_cpu.h"
-
-#define CHECK_CUDA(x) do { \
- if (!x.is_cuda()) { \
- fprintf(stderr, "%s must be CUDA tensor at %s:%d\n", #x, __FILE__, __LINE__); \
- exit(-1); \
- } \
-} while (0)
-#define CHECK_CONTIGUOUS(x) do { \
- if (!x.is_contiguous()) { \
- fprintf(stderr, "%s must be contiguous tensor at %s:%d\n", #x, __FILE__, __LINE__); \
- exit(-1); \
- } \
-} while (0)
-#define CHECK_INPUT(x) CHECK_CUDA(x);CHECK_CONTIGUOUS(x)
-
-inline float min(float a, float b){
- return a > b ? b : a;
-}
-
-inline float max(float a, float b){
- return a > b ? a : b;
-}
-
-const float EPS = 1e-8;
-struct Point {
- float x, y;
- __device__ Point() {}
- __device__ Point(double _x, double _y){
- x = _x, y = _y;
- }
-
- __device__ void set(float _x, float _y){
- x = _x; y = _y;
- }
-
- __device__ Point operator +(const Point &b)const{
- return Point(x + b.x, y + b.y);
- }
-
- __device__ Point operator -(const Point &b)const{
- return Point(x - b.x, y - b.y);
- }
-};
-
-inline float cross(const Point &a, const Point &b){
- return a.x * b.y - a.y * b.x;
-}
-
-inline float cross(const Point &p1, const Point &p2, const Point &p0){
- return (p1.x - p0.x) * (p2.y - p0.y) - (p2.x - p0.x) * (p1.y - p0.y);
-}
-
-inline int check_rect_cross(const Point &p1, const Point &p2, const Point &q1, const Point &q2){
- int ret = min(p1.x,p2.x) <= max(q1.x,q2.x) &&
- min(q1.x,q2.x) <= max(p1.x,p2.x) &&
- min(p1.y,p2.y) <= max(q1.y,q2.y) &&
- min(q1.y,q2.y) <= max(p1.y,p2.y);
- return ret;
-}
-
-inline int check_in_box2d(const float *box, const Point &p){
- //params: (7) [x, y, z, dx, dy, dz, heading]
- const float MARGIN = 1e-2;
-
- float center_x = box[0], center_y = box[1];
- float angle_cos = cos(-box[6]), angle_sin = sin(-box[6]); // rotate the point in the opposite direction of box
- float rot_x = (p.x - center_x) * angle_cos + (p.y - center_y) * (-angle_sin);
- float rot_y = (p.x - center_x) * angle_sin + (p.y - center_y) * angle_cos;
-
- return (fabs(rot_x) < box[3] / 2 + MARGIN && fabs(rot_y) < box[4] / 2 + MARGIN);
-}
-
-inline int intersection(const Point &p1, const Point &p0, const Point &q1, const Point &q0, Point &ans){
- // fast exclusion
- if (check_rect_cross(p0, p1, q0, q1) == 0) return 0;
-
- // check cross standing
- float s1 = cross(q0, p1, p0);
- float s2 = cross(p1, q1, p0);
- float s3 = cross(p0, q1, q0);
- float s4 = cross(q1, p1, q0);
-
- if (!(s1 * s2 > 0 && s3 * s4 > 0)) return 0;
-
- // calculate intersection of two lines
- float s5 = cross(q1, p1, p0);
- if(fabs(s5 - s1) > EPS){
- ans.x = (s5 * q0.x - s1 * q1.x) / (s5 - s1);
- ans.y = (s5 * q0.y - s1 * q1.y) / (s5 - s1);
-
- }
- else{
- float a0 = p0.y - p1.y, b0 = p1.x - p0.x, c0 = p0.x * p1.y - p1.x * p0.y;
- float a1 = q0.y - q1.y, b1 = q1.x - q0.x, c1 = q0.x * q1.y - q1.x * q0.y;
- float D = a0 * b1 - a1 * b0;
-
- ans.x = (b0 * c1 - b1 * c0) / D;
- ans.y = (a1 * c0 - a0 * c1) / D;
- }
-
- return 1;
-}
-
-inline void rotate_around_center(const Point ¢er, const float angle_cos, const float angle_sin, Point &p){
- float new_x = (p.x - center.x) * angle_cos + (p.y - center.y) * (-angle_sin) + center.x;
- float new_y = (p.x - center.x) * angle_sin + (p.y - center.y) * angle_cos + center.y;
- p.set(new_x, new_y);
-}
-
-inline int point_cmp(const Point &a, const Point &b, const Point ¢er){
- return atan2(a.y - center.y, a.x - center.x) > atan2(b.y - center.y, b.x - center.x);
-}
-
-inline float box_overlap(const float *box_a, const float *box_b){
- // params: box_a (7) [x, y, z, dx, dy, dz, heading]
- // params: box_b (7) [x, y, z, dx, dy, dz, heading]
-
-// float a_x1 = box_a[0], a_y1 = box_a[1], a_x2 = box_a[2], a_y2 = box_a[3], a_angle = box_a[4];
-// float b_x1 = box_b[0], b_y1 = box_b[1], b_x2 = box_b[2], b_y2 = box_b[3], b_angle = box_b[4];
- float a_angle = box_a[6], b_angle = box_b[6];
- float a_dx_half = box_a[3] / 2, b_dx_half = box_b[3] / 2, a_dy_half = box_a[4] / 2, b_dy_half = box_b[4] / 2;
- float a_x1 = box_a[0] - a_dx_half, a_y1 = box_a[1] - a_dy_half;
- float a_x2 = box_a[0] + a_dx_half, a_y2 = box_a[1] + a_dy_half;
- float b_x1 = box_b[0] - b_dx_half, b_y1 = box_b[1] - b_dy_half;
- float b_x2 = box_b[0] + b_dx_half, b_y2 = box_b[1] + b_dy_half;
-
- Point center_a(box_a[0], box_a[1]);
- Point center_b(box_b[0], box_b[1]);
-
- Point box_a_corners[5];
- box_a_corners[0].set(a_x1, a_y1);
- box_a_corners[1].set(a_x2, a_y1);
- box_a_corners[2].set(a_x2, a_y2);
- box_a_corners[3].set(a_x1, a_y2);
-
- Point box_b_corners[5];
- box_b_corners[0].set(b_x1, b_y1);
- box_b_corners[1].set(b_x2, b_y1);
- box_b_corners[2].set(b_x2, b_y2);
- box_b_corners[3].set(b_x1, b_y2);
-
- // get oriented corners
- float a_angle_cos = cos(a_angle), a_angle_sin = sin(a_angle);
- float b_angle_cos = cos(b_angle), b_angle_sin = sin(b_angle);
-
- for (int k = 0; k < 4; k++){
- rotate_around_center(center_a, a_angle_cos, a_angle_sin, box_a_corners[k]);
- rotate_around_center(center_b, b_angle_cos, b_angle_sin, box_b_corners[k]);
- }
-
- box_a_corners[4] = box_a_corners[0];
- box_b_corners[4] = box_b_corners[0];
-
- // get intersection of lines
- Point cross_points[16];
- Point poly_center;
- int cnt = 0, flag = 0;
-
- poly_center.set(0, 0);
- for (int i = 0; i < 4; i++){
- for (int j = 0; j < 4; j++){
- flag = intersection(box_a_corners[i + 1], box_a_corners[i], box_b_corners[j + 1], box_b_corners[j], cross_points[cnt]);
- if (flag){
- poly_center = poly_center + cross_points[cnt];
- cnt++;
- }
- }
- }
-
- // check corners
- for (int k = 0; k < 4; k++){
- if (check_in_box2d(box_a, box_b_corners[k])){
- poly_center = poly_center + box_b_corners[k];
- cross_points[cnt] = box_b_corners[k];
- cnt++;
- }
- if (check_in_box2d(box_b, box_a_corners[k])){
- poly_center = poly_center + box_a_corners[k];
- cross_points[cnt] = box_a_corners[k];
- cnt++;
- }
- }
-
- poly_center.x /= cnt;
- poly_center.y /= cnt;
-
- // sort the points of polygon
- Point temp;
- for (int j = 0; j < cnt - 1; j++){
- for (int i = 0; i < cnt - j - 1; i++){
- if (point_cmp(cross_points[i], cross_points[i + 1], poly_center)){
- temp = cross_points[i];
- cross_points[i] = cross_points[i + 1];
- cross_points[i + 1] = temp;
- }
- }
- }
-
- // get the overlap areas
- float area = 0;
- for (int k = 0; k < cnt - 1; k++){
- area += cross(cross_points[k] - cross_points[0], cross_points[k + 1] - cross_points[0]);
- }
-
- return fabs(area) / 2.0;
-}
-
-inline float iou_bev(const float *box_a, const float *box_b){
- // params: box_a (7) [x, y, z, dx, dy, dz, heading]
- // params: box_b (7) [x, y, z, dx, dy, dz, heading]
- float sa = box_a[3] * box_a[4];
- float sb = box_b[3] * box_b[4];
- float s_overlap = box_overlap(box_a, box_b);
- return s_overlap / fmaxf(sa + sb - s_overlap, EPS);
-}
-
-
-int boxes_iou_bev_cpu(at::Tensor boxes_a_tensor, at::Tensor boxes_b_tensor, at::Tensor ans_iou_tensor){
- // params boxes_a_tensor: (N, 7) [x, y, z, dx, dy, dz, heading]
- // params boxes_b_tensor: (M, 7) [x, y, z, dx, dy, dz, heading]
- // params ans_iou_tensor: (N, M)
-
- CHECK_CONTIGUOUS(boxes_a_tensor);
- CHECK_CONTIGUOUS(boxes_b_tensor);
-
- int num_boxes_a = boxes_a_tensor.size(0);
- int num_boxes_b = boxes_b_tensor.size(0);
- const float *boxes_a = boxes_a_tensor.data_ptr();
- const float *boxes_b = boxes_b_tensor.data_ptr();
- float *ans_iou = ans_iou_tensor.data_ptr();
-
- for (int i = 0; i < num_boxes_a; i++){
- for (int j = 0; j < num_boxes_b; j++){
- ans_iou[i * num_boxes_b + j] = iou_bev(boxes_a + i * 7, boxes_b + j * 7);
- }
- }
- return 1;
-}
diff --git a/OpenCOOD/opencood/pcdet_utils/iou3d_nms/src/iou3d_cpu.h b/OpenCOOD/opencood/pcdet_utils/iou3d_nms/src/iou3d_cpu.h
deleted file mode 100644
index 8835ee70..00000000
--- a/OpenCOOD/opencood/pcdet_utils/iou3d_nms/src/iou3d_cpu.h
+++ /dev/null
@@ -1,11 +0,0 @@
-#ifndef IOU3D_CPU_H
-#define IOU3D_CPU_H
-
-#include
-#include
-#include
-#include
-
-int boxes_iou_bev_cpu(at::Tensor boxes_a_tensor, at::Tensor boxes_b_tensor, at::Tensor ans_iou_tensor);
-
-#endif
diff --git a/OpenCOOD/opencood/pcdet_utils/iou3d_nms/src/iou3d_nms.cpp b/OpenCOOD/opencood/pcdet_utils/iou3d_nms/src/iou3d_nms.cpp
deleted file mode 100644
index 75dfe0c0..00000000
--- a/OpenCOOD/opencood/pcdet_utils/iou3d_nms/src/iou3d_nms.cpp
+++ /dev/null
@@ -1,186 +0,0 @@
-/*
-3D IoU Calculation and Rotated NMS(modified from 2D NMS written by others)
-Written by Shaoshuai Shi
-All Rights Reserved 2019-2020.
-*/
-
-#include
-#include
-#include
-#include
-#include
-#include "iou3d_nms.h"
-
-#define CHECK_CUDA(x) do { \
- if (!x.is_cuda()) { \
- fprintf(stderr, "%s must be CUDA tensor at %s:%d\n", #x, __FILE__, __LINE__); \
- exit(-1); \
- } \
-} while (0)
-#define CHECK_CONTIGUOUS(x) do { \
- if (!x.is_contiguous()) { \
- fprintf(stderr, "%s must be contiguous tensor at %s:%d\n", #x, __FILE__, __LINE__); \
- exit(-1); \
- } \
-} while (0)
-#define CHECK_INPUT(x) CHECK_CUDA(x);CHECK_CONTIGUOUS(x)
-
-#define DIVUP(m,n) ((m) / (n) + ((m) % (n) > 0))
-
-#define CHECK_ERROR(ans) { gpuAssert((ans), __FILE__, __LINE__); }
-inline void gpuAssert(cudaError_t code, const char *file, int line, bool abort=true)
-{
- if (code != cudaSuccess)
- {
- fprintf(stderr,"GPUassert: %s %s %d\n", cudaGetErrorString(code), file, line);
- if (abort) exit(code);
- }
-}
-
-const int THREADS_PER_BLOCK_NMS = sizeof(unsigned long long) * 8;
-
-
-void boxesoverlapLauncher(const int num_a, const float *boxes_a, const int num_b, const float *boxes_b, float *ans_overlap);
-void boxesioubevLauncher(const int num_a, const float *boxes_a, const int num_b, const float *boxes_b, float *ans_iou);
-void nmsLauncher(const float *boxes, unsigned long long * mask, int boxes_num, float nms_overlap_thresh);
-void nmsNormalLauncher(const float *boxes, unsigned long long * mask, int boxes_num, float nms_overlap_thresh);
-
-
-int boxes_overlap_bev_gpu(at::Tensor boxes_a, at::Tensor boxes_b, at::Tensor ans_overlap){
- // params boxes_a: (N, 7) [x, y, z, dx, dy, dz, heading]
- // params boxes_b: (M, 7) [x, y, z, dx, dy, dz, heading]
- // params ans_overlap: (N, M)
-
- CHECK_INPUT(boxes_a);
- CHECK_INPUT(boxes_b);
- CHECK_INPUT(ans_overlap);
-
- int num_a = boxes_a.size(0);
- int num_b = boxes_b.size(0);
-
- const float * boxes_a_data = boxes_a.data_ptr();
- const float * boxes_b_data = boxes_b.data_ptr();
- float * ans_overlap_data = ans_overlap.data_ptr();
-
- boxesoverlapLauncher(num_a, boxes_a_data, num_b, boxes_b_data, ans_overlap_data);
-
- return 1;
-}
-
-int boxes_iou_bev_gpu(at::Tensor boxes_a, at::Tensor boxes_b, at::Tensor ans_iou){
- // params boxes_a: (N, 7) [x, y, z, dx, dy, dz, heading]
- // params boxes_b: (M, 7) [x, y, z, dx, dy, dz, heading]
- // params ans_overlap: (N, M)
- CHECK_INPUT(boxes_a);
- CHECK_INPUT(boxes_b);
- CHECK_INPUT(ans_iou);
-
- int num_a = boxes_a.size(0);
- int num_b = boxes_b.size(0);
-
- const float * boxes_a_data = boxes_a.data_ptr();
- const float * boxes_b_data = boxes_b.data_ptr();
- float * ans_iou_data = ans_iou.data_ptr();
-
- boxesioubevLauncher(num_a, boxes_a_data, num_b, boxes_b_data, ans_iou_data);
-
- return 1;
-}
-
-int nms_gpu(at::Tensor boxes, at::Tensor keep, float nms_overlap_thresh){
- // params boxes: (N, 7) [x, y, z, dx, dy, dz, heading]
- // params keep: (N)
- CHECK_INPUT(boxes);
- CHECK_CONTIGUOUS(keep);
-
- int boxes_num = boxes.size(0);
- const float * boxes_data = boxes.data_ptr();
- long * keep_data = keep.data_ptr();
-
- const int col_blocks = DIVUP(boxes_num, THREADS_PER_BLOCK_NMS);
-
- unsigned long long *mask_data = NULL;
- CHECK_ERROR(cudaMalloc((void**)&mask_data, boxes_num * col_blocks * sizeof(unsigned long long)));
- nmsLauncher(boxes_data, mask_data, boxes_num, nms_overlap_thresh);
-
- // unsigned long long mask_cpu[boxes_num * col_blocks];
- // unsigned long long *mask_cpu = new unsigned long long [boxes_num * col_blocks];
- std::vector mask_cpu(boxes_num * col_blocks);
-
-// printf("boxes_num=%d, col_blocks=%d\n", boxes_num, col_blocks);
- CHECK_ERROR(cudaMemcpy(&mask_cpu[0], mask_data, boxes_num * col_blocks * sizeof(unsigned long long),
- cudaMemcpyDeviceToHost));
-
- cudaFree(mask_data);
-
- unsigned long long remv_cpu[col_blocks];
- memset(remv_cpu, 0, col_blocks * sizeof(unsigned long long));
-
- int num_to_keep = 0;
-
- for (int i = 0; i < boxes_num; i++){
- int nblock = i / THREADS_PER_BLOCK_NMS;
- int inblock = i % THREADS_PER_BLOCK_NMS;
-
- if (!(remv_cpu[nblock] & (1ULL << inblock))){
- keep_data[num_to_keep++] = i;
- unsigned long long *p = &mask_cpu[0] + i * col_blocks;
- for (int j = nblock; j < col_blocks; j++){
- remv_cpu[j] |= p[j];
- }
- }
- }
- if ( cudaSuccess != cudaGetLastError() ) printf( "Error!\n" );
-
- return num_to_keep;
-}
-
-
-int nms_normal_gpu(at::Tensor boxes, at::Tensor keep, float nms_overlap_thresh){
- // params boxes: (N, 7) [x, y, z, dx, dy, dz, heading]
- // params keep: (N)
-
- CHECK_INPUT(boxes);
- CHECK_CONTIGUOUS(keep);
-
- int boxes_num = boxes.size(0);
- const float * boxes_data = boxes.data_ptr();
- long * keep_data = keep.data_ptr();
-
- const int col_blocks = DIVUP(boxes_num, THREADS_PER_BLOCK_NMS);
-
- unsigned long long *mask_data = NULL;
- CHECK_ERROR(cudaMalloc((void**)&mask_data, boxes_num * col_blocks * sizeof(unsigned long long)));
- nmsNormalLauncher(boxes_data, mask_data, boxes_num, nms_overlap_thresh);
-
- // unsigned long long mask_cpu[boxes_num * col_blocks];
- // unsigned long long *mask_cpu = new unsigned long long [boxes_num * col_blocks];
- std::vector mask_cpu(boxes_num * col_blocks);
-
-// printf("boxes_num=%d, col_blocks=%d\n", boxes_num, col_blocks);
- CHECK_ERROR(cudaMemcpy(&mask_cpu[0], mask_data, boxes_num * col_blocks * sizeof(unsigned long long),
- cudaMemcpyDeviceToHost));
-
- cudaFree(mask_data);
-
- unsigned long long remv_cpu[col_blocks];
- memset(remv_cpu, 0, col_blocks * sizeof(unsigned long long));
-
- int num_to_keep = 0;
-
- for (int i = 0; i < boxes_num; i++){
- int nblock = i / THREADS_PER_BLOCK_NMS;
- int inblock = i % THREADS_PER_BLOCK_NMS;
-
- if (!(remv_cpu[nblock] & (1ULL << inblock))){
- keep_data[num_to_keep++] = i;
- unsigned long long *p = &mask_cpu[0] + i * col_blocks;
- for (int j = nblock; j < col_blocks; j++){
- remv_cpu[j] |= p[j];
- }
- }
- }
- if ( cudaSuccess != cudaGetLastError() ) printf( "Error!\n" );
-
- return num_to_keep;
-}
diff --git a/OpenCOOD/opencood/pcdet_utils/iou3d_nms/src/iou3d_nms.h b/OpenCOOD/opencood/pcdet_utils/iou3d_nms/src/iou3d_nms.h
deleted file mode 100644
index aa7ae0ed..00000000
--- a/OpenCOOD/opencood/pcdet_utils/iou3d_nms/src/iou3d_nms.h
+++ /dev/null
@@ -1,14 +0,0 @@
-#ifndef IOU3D_NMS_H
-#define IOU3D_NMS_H
-
-#include
-#include
-#include
-#include
-
-int boxes_overlap_bev_gpu(at::Tensor boxes_a, at::Tensor boxes_b, at::Tensor ans_overlap);
-int boxes_iou_bev_gpu(at::Tensor boxes_a, at::Tensor boxes_b, at::Tensor ans_iou);
-int nms_gpu(at::Tensor boxes, at::Tensor keep, float nms_overlap_thresh);
-int nms_normal_gpu(at::Tensor boxes, at::Tensor keep, float nms_overlap_thresh);
-
-#endif
diff --git a/OpenCOOD/opencood/pcdet_utils/iou3d_nms/src/iou3d_nms_api.cpp b/OpenCOOD/opencood/pcdet_utils/iou3d_nms/src/iou3d_nms_api.cpp
deleted file mode 100644
index 5a2d3a37..00000000
--- a/OpenCOOD/opencood/pcdet_utils/iou3d_nms/src/iou3d_nms_api.cpp
+++ /dev/null
@@ -1,17 +0,0 @@
-#include
-#include
-#include
-#include
-#include
-
-#include "iou3d_cpu.h"
-#include "iou3d_nms.h"
-
-
-PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
- m.def("boxes_overlap_bev_gpu", &boxes_overlap_bev_gpu, "oriented boxes overlap");
- m.def("boxes_iou_bev_gpu", &boxes_iou_bev_gpu, "oriented boxes iou");
- m.def("nms_gpu", &nms_gpu, "oriented nms gpu");
- m.def("nms_normal_gpu", &nms_normal_gpu, "nms gpu");
- m.def("boxes_iou_bev_cpu", &boxes_iou_bev_cpu, "oriented boxes iou");
-}
diff --git a/OpenCOOD/opencood/pcdet_utils/iou3d_nms/src/iou3d_nms_kernel.cu b/OpenCOOD/opencood/pcdet_utils/iou3d_nms/src/iou3d_nms_kernel.cu
deleted file mode 100644
index e5e305cd..00000000
--- a/OpenCOOD/opencood/pcdet_utils/iou3d_nms/src/iou3d_nms_kernel.cu
+++ /dev/null
@@ -1,414 +0,0 @@
-/*
-3D IoU Calculation and Rotated NMS(modified from 2D NMS written by others)
-Written by Shaoshuai Shi
-All Rights Reserved 2019-2020.
-*/
-
-
-#include
-#define THREADS_PER_BLOCK 16
-#define DIVUP(m, n) ((m) / (n) + ((m) % (n) > 0))
-
-// #define DEBUG
-const int THREADS_PER_BLOCK_NMS = sizeof(unsigned long long) * 8;
-const float EPS = 1e-8;
-struct Point {
- float x, y;
- __device__ Point() {}
- __device__ Point(double _x, double _y){
- x = _x, y = _y;
- }
-
- __device__ void set(float _x, float _y){
- x = _x; y = _y;
- }
-
- __device__ Point operator +(const Point &b)const{
- return Point(x + b.x, y + b.y);
- }
-
- __device__ Point operator -(const Point &b)const{
- return Point(x - b.x, y - b.y);
- }
-};
-
-__device__ inline float cross(const Point &a, const Point &b){
- return a.x * b.y - a.y * b.x;
-}
-
-__device__ inline float cross(const Point &p1, const Point &p2, const Point &p0){
- return (p1.x - p0.x) * (p2.y - p0.y) - (p2.x - p0.x) * (p1.y - p0.y);
-}
-
-__device__ int check_rect_cross(const Point &p1, const Point &p2, const Point &q1, const Point &q2){
- int ret = min(p1.x,p2.x) <= max(q1.x,q2.x) &&
- min(q1.x,q2.x) <= max(p1.x,p2.x) &&
- min(p1.y,p2.y) <= max(q1.y,q2.y) &&
- min(q1.y,q2.y) <= max(p1.y,p2.y);
- return ret;
-}
-
-__device__ inline int check_in_box2d(const float *box, const Point &p){
- //params: (7) [x, y, z, dx, dy, dz, heading]
- const float MARGIN = 1e-2;
-
- float center_x = box[0], center_y = box[1];
- float angle_cos = cos(-box[6]), angle_sin = sin(-box[6]); // rotate the point in the opposite direction of box
- float rot_x = (p.x - center_x) * angle_cos + (p.y - center_y) * (-angle_sin);
- float rot_y = (p.x - center_x) * angle_sin + (p.y - center_y) * angle_cos;
-
- return (fabs(rot_x) < box[3] / 2 + MARGIN && fabs(rot_y) < box[4] / 2 + MARGIN);
-}
-
-__device__ inline int intersection(const Point &p1, const Point &p0, const Point &q1, const Point &q0, Point &ans){
- // fast exclusion
- if (check_rect_cross(p0, p1, q0, q1) == 0) return 0;
-
- // check cross standing
- float s1 = cross(q0, p1, p0);
- float s2 = cross(p1, q1, p0);
- float s3 = cross(p0, q1, q0);
- float s4 = cross(q1, p1, q0);
-
- if (!(s1 * s2 > 0 && s3 * s4 > 0)) return 0;
-
- // calculate intersection of two lines
- float s5 = cross(q1, p1, p0);
- if(fabs(s5 - s1) > EPS){
- ans.x = (s5 * q0.x - s1 * q1.x) / (s5 - s1);
- ans.y = (s5 * q0.y - s1 * q1.y) / (s5 - s1);
-
- }
- else{
- float a0 = p0.y - p1.y, b0 = p1.x - p0.x, c0 = p0.x * p1.y - p1.x * p0.y;
- float a1 = q0.y - q1.y, b1 = q1.x - q0.x, c1 = q0.x * q1.y - q1.x * q0.y;
- float D = a0 * b1 - a1 * b0;
-
- ans.x = (b0 * c1 - b1 * c0) / D;
- ans.y = (a1 * c0 - a0 * c1) / D;
- }
-
- return 1;
-}
-
-__device__ inline void rotate_around_center(const Point ¢er, const float angle_cos, const float angle_sin, Point &p){
- float new_x = (p.x - center.x) * angle_cos + (p.y - center.y) * (-angle_sin) + center.x;
- float new_y = (p.x - center.x) * angle_sin + (p.y - center.y) * angle_cos + center.y;
- p.set(new_x, new_y);
-}
-
-__device__ inline int point_cmp(const Point &a, const Point &b, const Point ¢er){
- return atan2(a.y - center.y, a.x - center.x) > atan2(b.y - center.y, b.x - center.x);
-}
-
-__device__ inline float box_overlap(const float *box_a, const float *box_b){
- // params box_a: [x, y, z, dx, dy, dz, heading]
- // params box_b: [x, y, z, dx, dy, dz, heading]
-
- float a_angle = box_a[6], b_angle = box_b[6];
- float a_dx_half = box_a[3] / 2, b_dx_half = box_b[3] / 2, a_dy_half = box_a[4] / 2, b_dy_half = box_b[4] / 2;
- float a_x1 = box_a[0] - a_dx_half, a_y1 = box_a[1] - a_dy_half;
- float a_x2 = box_a[0] + a_dx_half, a_y2 = box_a[1] + a_dy_half;
- float b_x1 = box_b[0] - b_dx_half, b_y1 = box_b[1] - b_dy_half;
- float b_x2 = box_b[0] + b_dx_half, b_y2 = box_b[1] + b_dy_half;
-
- Point center_a(box_a[0], box_a[1]);
- Point center_b(box_b[0], box_b[1]);
-
-#ifdef DEBUG
- printf("a: (%.3f, %.3f, %.3f, %.3f, %.3f), b: (%.3f, %.3f, %.3f, %.3f, %.3f)\n", a_x1, a_y1, a_x2, a_y2, a_angle,
- b_x1, b_y1, b_x2, b_y2, b_angle);
- printf("center a: (%.3f, %.3f), b: (%.3f, %.3f)\n", center_a.x, center_a.y, center_b.x, center_b.y);
-#endif
-
- Point box_a_corners[5];
- box_a_corners[0].set(a_x1, a_y1);
- box_a_corners[1].set(a_x2, a_y1);
- box_a_corners[2].set(a_x2, a_y2);
- box_a_corners[3].set(a_x1, a_y2);
-
- Point box_b_corners[5];
- box_b_corners[0].set(b_x1, b_y1);
- box_b_corners[1].set(b_x2, b_y1);
- box_b_corners[2].set(b_x2, b_y2);
- box_b_corners[3].set(b_x1, b_y2);
-
- // get oriented corners
- float a_angle_cos = cos(a_angle), a_angle_sin = sin(a_angle);
- float b_angle_cos = cos(b_angle), b_angle_sin = sin(b_angle);
-
- for (int k = 0; k < 4; k++){
-#ifdef DEBUG
- printf("before corner %d: a(%.3f, %.3f), b(%.3f, %.3f) \n", k, box_a_corners[k].x, box_a_corners[k].y, box_b_corners[k].x, box_b_corners[k].y);
-#endif
- rotate_around_center(center_a, a_angle_cos, a_angle_sin, box_a_corners[k]);
- rotate_around_center(center_b, b_angle_cos, b_angle_sin, box_b_corners[k]);
-#ifdef DEBUG
- printf("corner %d: a(%.3f, %.3f), b(%.3f, %.3f) \n", k, box_a_corners[k].x, box_a_corners[k].y, box_b_corners[k].x, box_b_corners[k].y);
-#endif
- }
-
- box_a_corners[4] = box_a_corners[0];
- box_b_corners[4] = box_b_corners[0];
-
- // get intersection of lines
- Point cross_points[16];
- Point poly_center;
- int cnt = 0, flag = 0;
-
- poly_center.set(0, 0);
- for (int i = 0; i < 4; i++){
- for (int j = 0; j < 4; j++){
- flag = intersection(box_a_corners[i + 1], box_a_corners[i], box_b_corners[j + 1], box_b_corners[j], cross_points[cnt]);
- if (flag){
- poly_center = poly_center + cross_points[cnt];
- cnt++;
-#ifdef DEBUG
- printf("Cross points (%.3f, %.3f): a(%.3f, %.3f)->(%.3f, %.3f), b(%.3f, %.3f)->(%.3f, %.3f) \n",
- cross_points[cnt - 1].x, cross_points[cnt - 1].y,
- box_a_corners[i].x, box_a_corners[i].y, box_a_corners[i + 1].x, box_a_corners[i + 1].y,
- box_b_corners[i].x, box_b_corners[i].y, box_b_corners[i + 1].x, box_b_corners[i + 1].y);
-#endif
- }
- }
- }
-
- // check corners
- for (int k = 0; k < 4; k++){
- if (check_in_box2d(box_a, box_b_corners[k])){
- poly_center = poly_center + box_b_corners[k];
- cross_points[cnt] = box_b_corners[k];
- cnt++;
-#ifdef DEBUG
- printf("b corners in a: corner_b(%.3f, %.3f)", cross_points[cnt - 1].x, cross_points[cnt - 1].y);
-#endif
- }
- if (check_in_box2d(box_b, box_a_corners[k])){
- poly_center = poly_center + box_a_corners[k];
- cross_points[cnt] = box_a_corners[k];
- cnt++;
-#ifdef DEBUG
- printf("a corners in b: corner_a(%.3f, %.3f)", cross_points[cnt - 1].x, cross_points[cnt - 1].y);
-#endif
- }
- }
-
- poly_center.x /= cnt;
- poly_center.y /= cnt;
-
- // sort the points of polygon
- Point temp;
- for (int j = 0; j < cnt - 1; j++){
- for (int i = 0; i < cnt - j - 1; i++){
- if (point_cmp(cross_points[i], cross_points[i + 1], poly_center)){
- temp = cross_points[i];
- cross_points[i] = cross_points[i + 1];
- cross_points[i + 1] = temp;
- }
- }
- }
-
-#ifdef DEBUG
- printf("cnt=%d\n", cnt);
- for (int i = 0; i < cnt; i++){
- printf("All cross point %d: (%.3f, %.3f)\n", i, cross_points[i].x, cross_points[i].y);
- }
-#endif
-
- // get the overlap areas
- float area = 0;
- for (int k = 0; k < cnt - 1; k++){
- area += cross(cross_points[k] - cross_points[0], cross_points[k + 1] - cross_points[0]);
- }
-
- return fabs(area) / 2.0;
-}
-
-__device__ inline float iou_bev(const float *box_a, const float *box_b){
- // params box_a: [x, y, z, dx, dy, dz, heading]
- // params box_b: [x, y, z, dx, dy, dz, heading]
- float sa = box_a[3] * box_a[4];
- float sb = box_b[3] * box_b[4];
- float s_overlap = box_overlap(box_a, box_b);
- return s_overlap / fmaxf(sa + sb - s_overlap, EPS);
-}
-
-__global__ void boxes_overlap_kernel(const int num_a, const float *boxes_a, const int num_b, const float *boxes_b, float *ans_overlap){
- // params boxes_a: (N, 7) [x, y, z, dx, dy, dz, heading]
- // params boxes_b: (M, 7) [x, y, z, dx, dy, dz, heading]
- const int a_idx = blockIdx.y * THREADS_PER_BLOCK + threadIdx.y;
- const int b_idx = blockIdx.x * THREADS_PER_BLOCK + threadIdx.x;
-
- if (a_idx >= num_a || b_idx >= num_b){
- return;
- }
- const float * cur_box_a = boxes_a + a_idx * 7;
- const float * cur_box_b = boxes_b + b_idx * 7;
- float s_overlap = box_overlap(cur_box_a, cur_box_b);
- ans_overlap[a_idx * num_b + b_idx] = s_overlap;
-}
-
-__global__ void boxes_iou_bev_kernel(const int num_a, const float *boxes_a, const int num_b, const float *boxes_b, float *ans_iou){
- // params boxes_a: (N, 7) [x, y, z, dx, dy, dz, heading]
- // params boxes_b: (M, 7) [x, y, z, dx, dy, dz, heading]
- const int a_idx = blockIdx.y * THREADS_PER_BLOCK + threadIdx.y;
- const int b_idx = blockIdx.x * THREADS_PER_BLOCK + threadIdx.x;
-
- if (a_idx >= num_a || b_idx >= num_b){
- return;
- }
-
- const float * cur_box_a = boxes_a + a_idx * 7;
- const float * cur_box_b = boxes_b + b_idx * 7;
- float cur_iou_bev = iou_bev(cur_box_a, cur_box_b);
- ans_iou[a_idx * num_b + b_idx] = cur_iou_bev;
-}
-
-__global__ void nms_kernel(const int boxes_num, const float nms_overlap_thresh,
- const float *boxes, unsigned long long *mask){
- //params: boxes (N, 7) [x, y, z, dx, dy, dz, heading]
- //params: mask (N, N/THREADS_PER_BLOCK_NMS)
-
- const int row_start = blockIdx.y;
- const int col_start = blockIdx.x;
-
- // if (row_start > col_start) return;
-
- const int row_size = fminf(boxes_num - row_start * THREADS_PER_BLOCK_NMS, THREADS_PER_BLOCK_NMS);
- const int col_size = fminf(boxes_num - col_start * THREADS_PER_BLOCK_NMS, THREADS_PER_BLOCK_NMS);
-
- __shared__ float block_boxes[THREADS_PER_BLOCK_NMS * 7];
-
- if (threadIdx.x < col_size) {
- block_boxes[threadIdx.x * 7 + 0] = boxes[(THREADS_PER_BLOCK_NMS * col_start + threadIdx.x) * 7 + 0];
- block_boxes[threadIdx.x * 7 + 1] = boxes[(THREADS_PER_BLOCK_NMS * col_start + threadIdx.x) * 7 + 1];
- block_boxes[threadIdx.x * 7 + 2] = boxes[(THREADS_PER_BLOCK_NMS * col_start + threadIdx.x) * 7 + 2];
- block_boxes[threadIdx.x * 7 + 3] = boxes[(THREADS_PER_BLOCK_NMS * col_start + threadIdx.x) * 7 + 3];
- block_boxes[threadIdx.x * 7 + 4] = boxes[(THREADS_PER_BLOCK_NMS * col_start + threadIdx.x) * 7 + 4];
- block_boxes[threadIdx.x * 7 + 5] = boxes[(THREADS_PER_BLOCK_NMS * col_start + threadIdx.x) * 7 + 5];
- block_boxes[threadIdx.x * 7 + 6] = boxes[(THREADS_PER_BLOCK_NMS * col_start + threadIdx.x) * 7 + 6];
- }
- __syncthreads();
-
- if (threadIdx.x < row_size) {
- const int cur_box_idx = THREADS_PER_BLOCK_NMS * row_start + threadIdx.x;
- const float *cur_box = boxes + cur_box_idx * 7;
-
- int i = 0;
- unsigned long long t = 0;
- int start = 0;
- if (row_start == col_start) {
- start = threadIdx.x + 1;
- }
- for (i = start; i < col_size; i++) {
- if (iou_bev(cur_box, block_boxes + i * 7) > nms_overlap_thresh){
- t |= 1ULL << i;
- }
- }
- const int col_blocks = DIVUP(boxes_num, THREADS_PER_BLOCK_NMS);
- mask[cur_box_idx * col_blocks + col_start] = t;
- }
-}
-
-
-__device__ inline float iou_normal(float const * const a, float const * const b) {
- //params: a: [x, y, z, dx, dy, dz, heading]
- //params: b: [x, y, z, dx, dy, dz, heading]
-
- float left = fmaxf(a[0] - a[3] / 2, b[0] - b[3] / 2), right = fminf(a[0] + a[3] / 2, b[0] + b[3] / 2);
- float top = fmaxf(a[1] - a[4] / 2, b[1] - b[4] / 2), bottom = fminf(a[1] + a[4] / 2, b[1] + b[4] / 2);
- float width = fmaxf(right - left, 0.f), height = fmaxf(bottom - top, 0.f);
- float interS = width * height;
- float Sa = a[3] * a[4];
- float Sb = b[3] * b[4];
- return interS / fmaxf(Sa + Sb - interS, EPS);
-}
-
-
-__global__ void nms_normal_kernel(const int boxes_num, const float nms_overlap_thresh,
- const float *boxes, unsigned long long *mask){
- //params: boxes (N, 7) [x, y, z, dx, dy, dz, heading]
- //params: mask (N, N/THREADS_PER_BLOCK_NMS)
-
- const int row_start = blockIdx.y;
- const int col_start = blockIdx.x;
-
- // if (row_start > col_start) return;
-
- const int row_size = fminf(boxes_num - row_start * THREADS_PER_BLOCK_NMS, THREADS_PER_BLOCK_NMS);
- const int col_size = fminf(boxes_num - col_start * THREADS_PER_BLOCK_NMS, THREADS_PER_BLOCK_NMS);
-
- __shared__ float block_boxes[THREADS_PER_BLOCK_NMS * 7];
-
- if (threadIdx.x < col_size) {
- block_boxes[threadIdx.x * 7 + 0] = boxes[(THREADS_PER_BLOCK_NMS * col_start + threadIdx.x) * 7 + 0];
- block_boxes[threadIdx.x * 7 + 1] = boxes[(THREADS_PER_BLOCK_NMS * col_start + threadIdx.x) * 7 + 1];
- block_boxes[threadIdx.x * 7 + 2] = boxes[(THREADS_PER_BLOCK_NMS * col_start + threadIdx.x) * 7 + 2];
- block_boxes[threadIdx.x * 7 + 3] = boxes[(THREADS_PER_BLOCK_NMS * col_start + threadIdx.x) * 7 + 3];
- block_boxes[threadIdx.x * 7 + 4] = boxes[(THREADS_PER_BLOCK_NMS * col_start + threadIdx.x) * 7 + 4];
- block_boxes[threadIdx.x * 7 + 5] = boxes[(THREADS_PER_BLOCK_NMS * col_start + threadIdx.x) * 7 + 5];
- block_boxes[threadIdx.x * 7 + 6] = boxes[(THREADS_PER_BLOCK_NMS * col_start + threadIdx.x) * 7 + 6];
- }
- __syncthreads();
-
- if (threadIdx.x < row_size) {
- const int cur_box_idx = THREADS_PER_BLOCK_NMS * row_start + threadIdx.x;
- const float *cur_box = boxes + cur_box_idx * 7;
-
- int i = 0;
- unsigned long long t = 0;
- int start = 0;
- if (row_start == col_start) {
- start = threadIdx.x + 1;
- }
- for (i = start; i < col_size; i++) {
- if (iou_normal(cur_box, block_boxes + i * 7) > nms_overlap_thresh){
- t |= 1ULL << i;
- }
- }
- const int col_blocks = DIVUP(boxes_num, THREADS_PER_BLOCK_NMS);
- mask[cur_box_idx * col_blocks + col_start] = t;
- }
-}
-
-
-
-
-
-void boxesoverlapLauncher(const int num_a, const float *boxes_a, const int num_b, const float *boxes_b, float *ans_overlap){
-
- dim3 blocks(DIVUP(num_b, THREADS_PER_BLOCK), DIVUP(num_a, THREADS_PER_BLOCK)); // blockIdx.x(col), blockIdx.y(row)
- dim3 threads(THREADS_PER_BLOCK, THREADS_PER_BLOCK);
-
- boxes_overlap_kernel<<>>(num_a, boxes_a, num_b, boxes_b, ans_overlap);
-#ifdef DEBUG
- cudaDeviceSynchronize(); // for using printf in kernel function
-#endif
-}
-
-void boxesioubevLauncher(const int num_a, const float *boxes_a, const int num_b, const float *boxes_b, float *ans_iou){
-
- dim3 blocks(DIVUP(num_b, THREADS_PER_BLOCK), DIVUP(num_a, THREADS_PER_BLOCK)); // blockIdx.x(col), blockIdx.y(row)
- dim3 threads(THREADS_PER_BLOCK, THREADS_PER_BLOCK);
-
- boxes_iou_bev_kernel<<>>(num_a, boxes_a, num_b, boxes_b, ans_iou);
-#ifdef DEBUG
- cudaDeviceSynchronize(); // for using printf in kernel function
-#endif
-}
-
-
-void nmsLauncher(const float *boxes, unsigned long long * mask, int boxes_num, float nms_overlap_thresh){
- dim3 blocks(DIVUP(boxes_num, THREADS_PER_BLOCK_NMS),
- DIVUP(boxes_num, THREADS_PER_BLOCK_NMS));
- dim3 threads(THREADS_PER_BLOCK_NMS);
- nms_kernel<<>>(boxes_num, nms_overlap_thresh, boxes, mask);
-}
-
-
-void nmsNormalLauncher(const float *boxes, unsigned long long * mask, int boxes_num, float nms_overlap_thresh){
- dim3 blocks(DIVUP(boxes_num, THREADS_PER_BLOCK_NMS),
- DIVUP(boxes_num, THREADS_PER_BLOCK_NMS));
- dim3 threads(THREADS_PER_BLOCK_NMS);
- nms_normal_kernel<<>>(boxes_num, nms_overlap_thresh, boxes, mask);
-}
diff --git a/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_batch/pointnet2_utils.py b/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_batch/pointnet2_utils.py
deleted file mode 100644
index f15a4e5c..00000000
--- a/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_batch/pointnet2_utils.py
+++ /dev/null
@@ -1,116 +0,0 @@
-from typing import Tuple
-
-import torch
-import torch.nn as nn
-from torch.autograd import Function, Variable
-
-from opencood.pcdet_utils.pointnet2.pointnet2_batch import pointnet2_batch_cuda as pointnet2
-
-
-class GroupingOperation(Function):
- @staticmethod
- def forward(ctx, features: torch.Tensor, idx: torch.Tensor) -> torch.Tensor:
- """
- :param ctx:
- :param features: (B, C, N) tensor of features to group
- :param idx: (B, npoint, nsample) tensor containing the indicies of features to group with
- :return:
- output: (B, C, npoint, nsample) tensor
- """
- assert features.is_contiguous()
- assert idx.is_contiguous()
-
- B, nfeatures, nsample = idx.size()
- _, C, N = features.size()
- output = torch.cuda.FloatTensor(B, C, nfeatures, nsample)
-
- pointnet2.group_points_wrapper(B, C, N, nfeatures, nsample, features, idx, output)
-
- ctx.for_backwards = (idx, N)
- return output
-
- @staticmethod
- def backward(ctx, grad_out: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
- """
- :param ctx:
- :param grad_out: (B, C, npoint, nsample) tensor of the gradients of the output from forward
- :return:
- grad_features: (B, C, N) gradient of the features
- """
- idx, N = ctx.for_backwards
-
- B, C, npoint, nsample = grad_out.size()
- grad_features = Variable(torch.cuda.FloatTensor(B, C, N).zero_())
-
- grad_out_data = grad_out.data.contiguous()
- pointnet2.group_points_grad_wrapper(B, C, N, npoint, nsample, grad_out_data, idx, grad_features.data)
- return grad_features, None
-
-
-grouping_operation = GroupingOperation.apply
-
-
-class BallQuery(Function):
- @staticmethod
- def forward(ctx, radius: float, nsample: int, xyz: torch.Tensor, new_xyz: torch.Tensor) -> torch.Tensor:
- """
- :param ctx:
- :param radius: float, radius of the balls
- :param nsample: int, maximum number of features in the balls
- :param xyz: (B, N, 3) xyz coordinates of the features
- :param new_xyz: (B, npoint, 3) centers of the ball query
- :return:
- idx: (B, npoint, nsample) tensor with the indicies of the features that form the query balls
- """
- assert new_xyz.is_contiguous()
- assert xyz.is_contiguous()
-
- B, N, _ = xyz.size()
- npoint = new_xyz.size(1)
- idx = torch.cuda.IntTensor(B, npoint, nsample).zero_()
-
- pointnet2.ball_query_wrapper(B, N, npoint, radius, nsample, new_xyz, xyz, idx)
- return idx
-
- @staticmethod
- def backward(ctx, a=None):
- return None, None, None, None
-
-
-ball_query = BallQuery.apply
-
-
-class QueryAndGroup(nn.Module):
- def __init__(self, radius: float, nsample: int, use_xyz: bool = True):
- """
- :param radius: float, radius of ball
- :param nsample: int, maximum number of features to gather in the ball
- :param use_xyz:
- """
- super().__init__()
- self.radius, self.nsample, self.use_xyz = radius, nsample, use_xyz
-
- def forward(self, xyz: torch.Tensor, new_xyz: torch.Tensor, features: torch.Tensor = None) -> Tuple[torch.Tensor]:
- """
- :param xyz: (B, N, 3) xyz coordinates of the features
- :param new_xyz: (B, npoint, 3) centroids
- :param features: (B, C, N) descriptors of the features
- :return:
- new_features: (B, 3 + C, npoint, nsample)
- """
- idx = ball_query(self.radius, self.nsample, xyz, new_xyz)
- xyz_trans = xyz.transpose(1, 2).contiguous()
- grouped_xyz = grouping_operation(xyz_trans, idx) # (B, 3, npoint, nsample)
- grouped_xyz -= new_xyz.transpose(1, 2).unsqueeze(-1)
-
- if features is not None:
- grouped_features = grouping_operation(features, idx)
- if self.use_xyz:
- new_features = torch.cat([grouped_xyz, grouped_features], dim=1) # (B, C + 3, npoint, nsample)
- else:
- new_features = grouped_features
- else:
- assert self.use_xyz, "Cannot have not features and not use xyz as a feature!"
- new_features = grouped_xyz
-
- return new_features
diff --git a/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_batch/src/ball_query.cpp b/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_batch/src/ball_query.cpp
deleted file mode 100644
index cf1db8c6..00000000
--- a/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_batch/src/ball_query.cpp
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
-Batch version of ball query, modified from the original implementation of official PointNet++ codes.
-Written by Shaoshuai Shi
-All Rights Reserved 2018.
-*/
-
-#include
-#include
-#include
-#include
-#include "ball_query_gpu.h"
-
-#define CHECK_CUDA(x) TORCH_CHECK(x.is_cuda(), #x " must be a CUDA tensor")
-#define CHECK_CONTIGUOUS(x) TORCH_CHECK(x.is_contiguous(), #x " must be contiguous")
-#define CHECK_INPUT(x) CHECK_CUDA(x); CHECK_CONTIGUOUS(x)
-
-int ball_query_wrapper_fast(int b, int n, int m, float radius, int nsample,
- at::Tensor new_xyz_tensor, at::Tensor xyz_tensor, at::Tensor idx_tensor) {
-
- CHECK_INPUT(new_xyz_tensor);
- CHECK_INPUT(xyz_tensor);
- CHECK_INPUT(idx_tensor);
-
- const float* new_xyz = new_xyz_tensor.data_ptr();
- const float* xyz = xyz_tensor.data_ptr();
- int* idx = idx_tensor.data_ptr();
-
- ball_query_kernel_launcher_fast(b, n, m, radius, nsample, new_xyz, xyz, idx);
- return 1;
-}
diff --git a/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_batch/src/ball_query_gpu.cu b/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_batch/src/ball_query_gpu.cu
deleted file mode 100644
index 870ca123..00000000
--- a/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_batch/src/ball_query_gpu.cu
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
-batch version of ball query, modified from the original implementation of official PointNet++ codes.
-Written by Shaoshuai Shi
-All Rights Reserved 2018.
-*/
-
-#include
-#include
-#include
-
-#include "ball_query_gpu.h"
-#include "cuda_utils.h"
-
-
-__global__ void ball_query_kernel_fast(int b, int n, int m, float radius, int nsample,
- const float *__restrict__ new_xyz, const float *__restrict__ xyz, int *__restrict__ idx) {
- // new_xyz: (B, M, 3)
- // xyz: (B, N, 3)
- // output:
- // idx: (B, M, nsample)
- int bs_idx = blockIdx.y;
- int pt_idx = blockIdx.x * blockDim.x + threadIdx.x;
- if (bs_idx >= b || pt_idx >= m) return;
-
- new_xyz += bs_idx * m * 3 + pt_idx * 3;
- xyz += bs_idx * n * 3;
- idx += bs_idx * m * nsample + pt_idx * nsample;
-
- float radius2 = radius * radius;
- float new_x = new_xyz[0];
- float new_y = new_xyz[1];
- float new_z = new_xyz[2];
-
- int cnt = 0;
- for (int k = 0; k < n; ++k) {
- float x = xyz[k * 3 + 0];
- float y = xyz[k * 3 + 1];
- float z = xyz[k * 3 + 2];
- float d2 = (new_x - x) * (new_x - x) + (new_y - y) * (new_y - y) + (new_z - z) * (new_z - z);
- if (d2 < radius2){
- if (cnt == 0){
- for (int l = 0; l < nsample; ++l) {
- idx[l] = k;
- }
- }
- idx[cnt] = k;
- ++cnt;
- if (cnt >= nsample) break;
- }
- }
-}
-
-
-void ball_query_kernel_launcher_fast(int b, int n, int m, float radius, int nsample, \
- const float *new_xyz, const float *xyz, int *idx) {
- // new_xyz: (B, M, 3)
- // xyz: (B, N, 3)
- // output:
- // idx: (B, M, nsample)
-
- cudaError_t err;
-
- dim3 blocks(DIVUP(m, THREADS_PER_BLOCK), b); // blockIdx.x(col), blockIdx.y(row)
- dim3 threads(THREADS_PER_BLOCK);
-
- ball_query_kernel_fast<<>>(b, n, m, radius, nsample, new_xyz, xyz, idx);
- // cudaDeviceSynchronize(); // for using printf in kernel function
- err = cudaGetLastError();
- if (cudaSuccess != err) {
- fprintf(stderr, "CUDA kernel failed : %s\n", cudaGetErrorString(err));
- exit(-1);
- }
-}
diff --git a/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_batch/src/ball_query_gpu.h b/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_batch/src/ball_query_gpu.h
deleted file mode 100644
index 7b181984..00000000
--- a/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_batch/src/ball_query_gpu.h
+++ /dev/null
@@ -1,15 +0,0 @@
-#ifndef _BALL_QUERY_GPU_H
-#define _BALL_QUERY_GPU_H
-
-#include
-#include
-#include
-#include
-
-int ball_query_wrapper_fast(int b, int n, int m, float radius, int nsample,
- at::Tensor new_xyz_tensor, at::Tensor xyz_tensor, at::Tensor idx_tensor);
-
-void ball_query_kernel_launcher_fast(int b, int n, int m, float radius, int nsample,
- const float *xyz, const float *new_xyz, int *idx);
-
-#endif
diff --git a/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_batch/src/cuda_utils.h b/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_batch/src/cuda_utils.h
deleted file mode 100644
index 7fe27969..00000000
--- a/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_batch/src/cuda_utils.h
+++ /dev/null
@@ -1,15 +0,0 @@
-#ifndef _CUDA_UTILS_H
-#define _CUDA_UTILS_H
-
-#include
-
-#define TOTAL_THREADS 1024
-#define THREADS_PER_BLOCK 256
-#define DIVUP(m,n) ((m) / (n) + ((m) % (n) > 0))
-
-inline int opt_n_threads(int work_size) {
- const int pow_2 = std::log(static_cast(work_size)) / std::log(2.0);
-
- return max(min(1 << pow_2, TOTAL_THREADS), 1);
-}
-#endif
diff --git a/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_batch/src/group_points.cpp b/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_batch/src/group_points.cpp
deleted file mode 100644
index 280cff5c..00000000
--- a/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_batch/src/group_points.cpp
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
-Batch version of point grouping, modified from the original implementation of official PointNet++ codes.
-Written by Shaoshuai Shi
-All Rights Reserved 2018.
-*/
-
-#include
-#include
-#include
-#include
-
-#include "group_points_gpu.h"
-
-#define CHECK_CUDA(x) TORCH_CHECK(x.is_cuda(), #x " must be a CUDA tensor")
-#define CHECK_CONTIGUOUS(x) TORCH_CHECK(x.is_contiguous(), #x " must be contiguous")
-#define CHECK_INPUT(x) CHECK_CUDA(x); CHECK_CONTIGUOUS(x)
-
-
-int group_points_grad_wrapper_fast(int b, int c, int n, int npoints, int nsample,
- at::Tensor grad_out_tensor, at::Tensor idx_tensor, at::Tensor grad_points_tensor) {
-
- CHECK_INPUT(grad_out_tensor);
- CHECK_INPUT(idx_tensor);
- CHECK_INPUT(grad_points_tensor);
-
- float* grad_points = grad_points_tensor.data_ptr();
- const int* idx = idx_tensor.data_ptr();
- const float* grad_out = grad_out_tensor.data_ptr();
-
- group_points_grad_kernel_launcher_fast(b, c, n, npoints, nsample, grad_out, idx, grad_points);
- return 1;
-}
-
-
-int group_points_wrapper_fast(int b, int c, int n, int npoints, int nsample,
- at::Tensor points_tensor, at::Tensor idx_tensor, at::Tensor out_tensor) {
-
- CHECK_INPUT(points_tensor);
- CHECK_INPUT(idx_tensor);
- CHECK_INPUT(out_tensor);
-
- const float* points = points_tensor.data_ptr();
- const int* idx = idx_tensor.data_ptr();
- float* out = out_tensor.data_ptr();
-
- group_points_kernel_launcher_fast(b, c, n, npoints, nsample, points, idx, out);
- return 1;
-}
diff --git a/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_batch/src/group_points_gpu.cu b/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_batch/src/group_points_gpu.cu
deleted file mode 100644
index a13c0a34..00000000
--- a/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_batch/src/group_points_gpu.cu
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
-batch version of point grouping, modified from the original implementation of official PointNet++ codes.
-Written by Shaoshuai Shi
-All Rights Reserved 2018.
-*/
-
-#include
-#include
-
-#include "cuda_utils.h"
-#include "group_points_gpu.h"
-
-
-__global__ void group_points_grad_kernel_fast(int b, int c, int n, int npoints, int nsample,
- const float *__restrict__ grad_out, const int *__restrict__ idx, float *__restrict__ grad_points) {
- // grad_out: (B, C, npoints, nsample)
- // idx: (B, npoints, nsample)
- // output:
- // grad_points: (B, C, N)
- int bs_idx = blockIdx.z;
- int c_idx = blockIdx.y;
- int index = blockIdx.x * blockDim.x + threadIdx.x;
- int pt_idx = index / nsample;
- if (bs_idx >= b || c_idx >= c || pt_idx >= npoints) return;
-
- int sample_idx = index % nsample;
- grad_out += bs_idx * c * npoints * nsample + c_idx * npoints * nsample + pt_idx * nsample + sample_idx;
- idx += bs_idx * npoints * nsample + pt_idx * nsample + sample_idx;
-
- atomicAdd(grad_points + bs_idx * c * n + c_idx * n + idx[0] , grad_out[0]);
-}
-
-void group_points_grad_kernel_launcher_fast(int b, int c, int n, int npoints, int nsample,
- const float *grad_out, const int *idx, float *grad_points) {
- // grad_out: (B, C, npoints, nsample)
- // idx: (B, npoints, nsample)
- // output:
- // grad_points: (B, C, N)
- cudaError_t err;
- dim3 blocks(DIVUP(npoints * nsample, THREADS_PER_BLOCK), c, b); // blockIdx.x(col), blockIdx.y(row)
- dim3 threads(THREADS_PER_BLOCK);
-
- group_points_grad_kernel_fast<<>>(b, c, n, npoints, nsample, grad_out, idx, grad_points);
-
- err = cudaGetLastError();
- if (cudaSuccess != err) {
- fprintf(stderr, "CUDA kernel failed : %s\n", cudaGetErrorString(err));
- exit(-1);
- }
-}
-
-
-__global__ void group_points_kernel_fast(int b, int c, int n, int npoints, int nsample,
- const float *__restrict__ points, const int *__restrict__ idx, float *__restrict__ out) {
- // points: (B, C, N)
- // idx: (B, npoints, nsample)
- // output:
- // out: (B, C, npoints, nsample)
- int bs_idx = blockIdx.z;
- int c_idx = blockIdx.y;
- int index = blockIdx.x * blockDim.x + threadIdx.x;
- int pt_idx = index / nsample;
- if (bs_idx >= b || c_idx >= c || pt_idx >= npoints) return;
-
- int sample_idx = index % nsample;
-
- idx += bs_idx * npoints * nsample + pt_idx * nsample + sample_idx;
- int in_idx = bs_idx * c * n + c_idx * n + idx[0];
- int out_idx = bs_idx * c * npoints * nsample + c_idx * npoints * nsample + pt_idx * nsample + sample_idx;
-
- out[out_idx] = points[in_idx];
-}
-
-
-void group_points_kernel_launcher_fast(int b, int c, int n, int npoints, int nsample,
- const float *points, const int *idx, float *out) {
- // points: (B, C, N)
- // idx: (B, npoints, nsample)
- // output:
- // out: (B, C, npoints, nsample)
- cudaError_t err;
- dim3 blocks(DIVUP(npoints * nsample, THREADS_PER_BLOCK), c, b); // blockIdx.x(col), blockIdx.y(row)
- dim3 threads(THREADS_PER_BLOCK);
-
- group_points_kernel_fast<<>>(b, c, n, npoints, nsample, points, idx, out);
- // cudaDeviceSynchronize(); // for using printf in kernel function
- err = cudaGetLastError();
- if (cudaSuccess != err) {
- fprintf(stderr, "CUDA kernel failed : %s\n", cudaGetErrorString(err));
- exit(-1);
- }
-}
diff --git a/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_batch/src/group_points_gpu.h b/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_batch/src/group_points_gpu.h
deleted file mode 100644
index 6eb72259..00000000
--- a/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_batch/src/group_points_gpu.h
+++ /dev/null
@@ -1,22 +0,0 @@
-#ifndef _GROUP_POINTS_GPU_H
-#define _GROUP_POINTS_GPU_H
-
-#include
-#include
-#include
-#include
-
-
-int group_points_wrapper_fast(int b, int c, int n, int npoints, int nsample,
- at::Tensor points_tensor, at::Tensor idx_tensor, at::Tensor out_tensor);
-
-void group_points_kernel_launcher_fast(int b, int c, int n, int npoints, int nsample,
- const float *points, const int *idx, float *out);
-
-int group_points_grad_wrapper_fast(int b, int c, int n, int npoints, int nsample,
- at::Tensor grad_out_tensor, at::Tensor idx_tensor, at::Tensor grad_points_tensor);
-
-void group_points_grad_kernel_launcher_fast(int b, int c, int n, int npoints, int nsample,
- const float *grad_out, const int *idx, float *grad_points);
-
-#endif
diff --git a/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_batch/src/interpolate.cpp b/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_batch/src/interpolate.cpp
deleted file mode 100644
index 2b9581d3..00000000
--- a/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_batch/src/interpolate.cpp
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
-Batch version of point interpolation, modified from the original implementation of official PointNet++ codes.
-Written by Shaoshuai Shi
-All Rights Reserved 2018.
-*/
-
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include "interpolate_gpu.h"
-
-
-#define CHECK_CUDA(x) TORCH_CHECK(x.is_cuda(), #x " must be a CUDA tensor")
-#define CHECK_CONTIGUOUS(x) TORCH_CHECK(x.is_contiguous(), #x " must be contiguous")
-#define CHECK_INPUT(x) CHECK_CUDA(x); CHECK_CONTIGUOUS(x)
-
-
-void three_nn_wrapper_fast(int b, int n, int m, at::Tensor unknown_tensor,
- at::Tensor known_tensor, at::Tensor dist2_tensor, at::Tensor idx_tensor) {
-
- CHECK_INPUT(unknown_tensor);
- CHECK_INPUT(known_tensor);
- CHECK_INPUT(dist2_tensor);
- CHECK_INPUT(idx_tensor);
-
- const float* unknown = unknown_tensor.data_ptr();
- const float* known = known_tensor.data_ptr();
- float* dist2 = dist2_tensor.data_ptr();
- int* idx = idx_tensor.data_ptr();
-
- three_nn_kernel_launcher_fast(b, n, m, unknown, known, dist2, idx);
-}
-
-
-void three_interpolate_wrapper_fast(int b, int c, int m, int n,
- at::Tensor points_tensor,
- at::Tensor idx_tensor,
- at::Tensor weight_tensor,
- at::Tensor out_tensor) {
-
- CHECK_INPUT(points_tensor);
- CHECK_INPUT(idx_tensor);
- CHECK_INPUT(weight_tensor);
- CHECK_INPUT(out_tensor);
-
- const float* points = points_tensor.data_ptr();
- const float* weight = weight_tensor.data_ptr();
- float* out = out_tensor.data_ptr();
- const int* idx = idx_tensor.data_ptr();
-
- three_interpolate_kernel_launcher_fast(b, c, m, n, points, idx, weight, out);
-}
-
-
-void three_interpolate_grad_wrapper_fast(int b, int c, int n, int m,
- at::Tensor grad_out_tensor,
- at::Tensor idx_tensor,
- at::Tensor weight_tensor,
- at::Tensor grad_points_tensor) {
-
- CHECK_INPUT(grad_out_tensor);
- CHECK_INPUT(idx_tensor);
- CHECK_INPUT(weight_tensor);
- CHECK_INPUT(grad_points_tensor);
-
- const float* grad_out = grad_out_tensor.data_ptr();
- const float* weight = weight_tensor.data_ptr();
- float* grad_points = grad_points_tensor.data_ptr();
- const int* idx = idx_tensor.data_ptr();
-
- three_interpolate_grad_kernel_launcher_fast(b, c, n, m, grad_out, idx, weight, grad_points);
-}
diff --git a/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_batch/src/interpolate_gpu.cu b/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_batch/src/interpolate_gpu.cu
deleted file mode 100644
index d94e0778..00000000
--- a/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_batch/src/interpolate_gpu.cu
+++ /dev/null
@@ -1,168 +0,0 @@
-/*
-batch version of point interpolation, modified from the original implementation of official PointNet++ codes.
-Written by Shaoshuai Shi
-All Rights Reserved 2018.
-*/
-
-
-#include
-#include
-#include
-
-#include "cuda_utils.h"
-#include "interpolate_gpu.h"
-
-
-__global__ void three_nn_kernel_fast(int b, int n, int m, const float *__restrict__ unknown,
- const float *__restrict__ known, float *__restrict__ dist2, int *__restrict__ idx) {
- // unknown: (B, N, 3)
- // known: (B, M, 3)
- // output:
- // dist2: (B, N, 3)
- // idx: (B, N, 3)
-
- int bs_idx = blockIdx.y;
- int pt_idx = blockIdx.x * blockDim.x + threadIdx.x;
- if (bs_idx >= b || pt_idx >= n) return;
-
- unknown += bs_idx * n * 3 + pt_idx * 3;
- known += bs_idx * m * 3;
- dist2 += bs_idx * n * 3 + pt_idx * 3;
- idx += bs_idx * n * 3 + pt_idx * 3;
-
- float ux = unknown[0];
- float uy = unknown[1];
- float uz = unknown[2];
-
- double best1 = 1e40, best2 = 1e40, best3 = 1e40;
- int besti1 = 0, besti2 = 0, besti3 = 0;
- for (int k = 0; k < m; ++k) {
- float x = known[k * 3 + 0];
- float y = known[k * 3 + 1];
- float z = known[k * 3 + 2];
- float d = (ux - x) * (ux - x) + (uy - y) * (uy - y) + (uz - z) * (uz - z);
- if (d < best1) {
- best3 = best2; besti3 = besti2;
- best2 = best1; besti2 = besti1;
- best1 = d; besti1 = k;
- }
- else if (d < best2) {
- best3 = best2; besti3 = besti2;
- best2 = d; besti2 = k;
- }
- else if (d < best3) {
- best3 = d; besti3 = k;
- }
- }
- dist2[0] = best1; dist2[1] = best2; dist2[2] = best3;
- idx[0] = besti1; idx[1] = besti2; idx[2] = besti3;
-}
-
-
-void three_nn_kernel_launcher_fast(int b, int n, int m, const float *unknown,
- const float *known, float *dist2, int *idx) {
- // unknown: (B, N, 3)
- // known: (B, M, 3)
- // output:
- // dist2: (B, N, 3)
- // idx: (B, N, 3)
-
- cudaError_t err;
- dim3 blocks(DIVUP(n, THREADS_PER_BLOCK), b); // blockIdx.x(col), blockIdx.y(row)
- dim3 threads(THREADS_PER_BLOCK);
-
- three_nn_kernel_fast<<>>(b, n, m, unknown, known, dist2, idx);
-
- err = cudaGetLastError();
- if (cudaSuccess != err) {
- fprintf(stderr, "CUDA kernel failed : %s\n", cudaGetErrorString(err));
- exit(-1);
- }
-}
-
-
-__global__ void three_interpolate_kernel_fast(int b, int c, int m, int n, const float *__restrict__ points,
- const int *__restrict__ idx, const float *__restrict__ weight, float *__restrict__ out) {
- // points: (B, C, M)
- // idx: (B, N, 3)
- // weight: (B, N, 3)
- // output:
- // out: (B, C, N)
-
- int bs_idx = blockIdx.z;
- int c_idx = blockIdx.y;
- int pt_idx = blockIdx.x * blockDim.x + threadIdx.x;
-
- if (bs_idx >= b || c_idx >= c || pt_idx >= n) return;
-
- weight += bs_idx * n * 3 + pt_idx * 3;
- points += bs_idx * c * m + c_idx * m;
- idx += bs_idx * n * 3 + pt_idx * 3;
- out += bs_idx * c * n + c_idx * n;
-
- out[pt_idx] = weight[0] * points[idx[0]] + weight[1] * points[idx[1]] + weight[2] * points[idx[2]];
-}
-
-void three_interpolate_kernel_launcher_fast(int b, int c, int m, int n,
- const float *points, const int *idx, const float *weight, float *out) {
- // points: (B, C, M)
- // idx: (B, N, 3)
- // weight: (B, N, 3)
- // output:
- // out: (B, C, N)
-
- cudaError_t err;
- dim3 blocks(DIVUP(n, THREADS_PER_BLOCK), c, b); // blockIdx.x(col), blockIdx.y(row)
- dim3 threads(THREADS_PER_BLOCK);
- three_interpolate_kernel_fast<<>>(b, c, m, n, points, idx, weight, out);
-
- err = cudaGetLastError();
- if (cudaSuccess != err) {
- fprintf(stderr, "CUDA kernel failed : %s\n", cudaGetErrorString(err));
- exit(-1);
- }
-}
-
-
-__global__ void three_interpolate_grad_kernel_fast(int b, int c, int n, int m, const float *__restrict__ grad_out,
- const int *__restrict__ idx, const float *__restrict__ weight, float *__restrict__ grad_points) {
- // grad_out: (B, C, N)
- // weight: (B, N, 3)
- // output:
- // grad_points: (B, C, M)
-
- int bs_idx = blockIdx.z;
- int c_idx = blockIdx.y;
- int pt_idx = blockIdx.x * blockDim.x + threadIdx.x;
-
- if (bs_idx >= b || c_idx >= c || pt_idx >= n) return;
-
- grad_out += bs_idx * c * n + c_idx * n + pt_idx;
- weight += bs_idx * n * 3 + pt_idx * 3;
- grad_points += bs_idx * c * m + c_idx * m;
- idx += bs_idx * n * 3 + pt_idx * 3;
-
-
- atomicAdd(grad_points + idx[0], grad_out[0] * weight[0]);
- atomicAdd(grad_points + idx[1], grad_out[0] * weight[1]);
- atomicAdd(grad_points + idx[2], grad_out[0] * weight[2]);
-}
-
-void three_interpolate_grad_kernel_launcher_fast(int b, int c, int n, int m, const float *grad_out,
- const int *idx, const float *weight, float *grad_points) {
- // grad_out: (B, C, N)
- // weight: (B, N, 3)
- // output:
- // grad_points: (B, C, M)
-
- cudaError_t err;
- dim3 blocks(DIVUP(n, THREADS_PER_BLOCK), c, b); // blockIdx.x(col), blockIdx.y(row)
- dim3 threads(THREADS_PER_BLOCK);
- three_interpolate_grad_kernel_fast<<>>(b, c, n, m, grad_out, idx, weight, grad_points);
-
- err = cudaGetLastError();
- if (cudaSuccess != err) {
- fprintf(stderr, "CUDA kernel failed : %s\n", cudaGetErrorString(err));
- exit(-1);
- }
-}
diff --git a/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_batch/src/interpolate_gpu.h b/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_batch/src/interpolate_gpu.h
deleted file mode 100644
index 19d41eb1..00000000
--- a/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_batch/src/interpolate_gpu.h
+++ /dev/null
@@ -1,30 +0,0 @@
-#ifndef _INTERPOLATE_GPU_H
-#define _INTERPOLATE_GPU_H
-
-#include
-#include
-#include
-#include
-
-
-void three_nn_wrapper_fast(int b, int n, int m, at::Tensor unknown_tensor,
- at::Tensor known_tensor, at::Tensor dist2_tensor, at::Tensor idx_tensor);
-
-void three_nn_kernel_launcher_fast(int b, int n, int m, const float *unknown,
- const float *known, float *dist2, int *idx);
-
-
-void three_interpolate_wrapper_fast(int b, int c, int m, int n, at::Tensor points_tensor,
- at::Tensor idx_tensor, at::Tensor weight_tensor, at::Tensor out_tensor);
-
-void three_interpolate_kernel_launcher_fast(int b, int c, int m, int n,
- const float *points, const int *idx, const float *weight, float *out);
-
-
-void three_interpolate_grad_wrapper_fast(int b, int c, int n, int m, at::Tensor grad_out_tensor,
- at::Tensor idx_tensor, at::Tensor weight_tensor, at::Tensor grad_points_tensor);
-
-void three_interpolate_grad_kernel_launcher_fast(int b, int c, int n, int m, const float *grad_out,
- const int *idx, const float *weight, float *grad_points);
-
-#endif
diff --git a/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_batch/src/pointnet2_api.cpp b/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_batch/src/pointnet2_api.cpp
deleted file mode 100644
index 10038bcb..00000000
--- a/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_batch/src/pointnet2_api.cpp
+++ /dev/null
@@ -1,24 +0,0 @@
-#include
-#include
-
-#include "ball_query_gpu.h"
-#include "group_points_gpu.h"
-#include "sampling_gpu.h"
-#include "interpolate_gpu.h"
-
-
-PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
- m.def("ball_query_wrapper", &ball_query_wrapper_fast, "ball_query_wrapper_fast");
-
- m.def("group_points_wrapper", &group_points_wrapper_fast, "group_points_wrapper_fast");
- m.def("group_points_grad_wrapper", &group_points_grad_wrapper_fast, "group_points_grad_wrapper_fast");
-
- m.def("gather_points_wrapper", &gather_points_wrapper_fast, "gather_points_wrapper_fast");
- m.def("gather_points_grad_wrapper", &gather_points_grad_wrapper_fast, "gather_points_grad_wrapper_fast");
-
- m.def("furthest_point_sampling_wrapper", &furthest_point_sampling_wrapper, "furthest_point_sampling_wrapper");
-
- m.def("three_nn_wrapper", &three_nn_wrapper_fast, "three_nn_wrapper_fast");
- m.def("three_interpolate_wrapper", &three_interpolate_wrapper_fast, "three_interpolate_wrapper_fast");
- m.def("three_interpolate_grad_wrapper", &three_interpolate_grad_wrapper_fast, "three_interpolate_grad_wrapper_fast");
-}
diff --git a/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_batch/src/sampling.cpp b/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_batch/src/sampling.cpp
deleted file mode 100644
index 27b91d49..00000000
--- a/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_batch/src/sampling.cpp
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
-Batch version of point sampling and gathering, modified from the original implementation of official PointNet++ codes.
-Written by Shaoshuai Shi
-All Rights Reserved 2018.
-*/
-
-#include
-#include
-#include
-
-#include "sampling_gpu.h"
-
-
-#define CHECK_CUDA(x) TORCH_CHECK(x.is_cuda(), #x " must be a CUDA tensor")
-#define CHECK_CONTIGUOUS(x) TORCH_CHECK(x.is_contiguous(), #x " must be contiguous")
-#define CHECK_INPUT(x) CHECK_CUDA(x); CHECK_CONTIGUOUS(x)
-
-
-int gather_points_wrapper_fast(int b, int c, int n, int npoints,
- at::Tensor points_tensor, at::Tensor idx_tensor, at::Tensor out_tensor){
-
- CHECK_INPUT(points_tensor);
- CHECK_INPUT(idx_tensor);
- CHECK_INPUT(out_tensor);
-
- const float* points = points_tensor.data_ptr();
- const int* idx = idx_tensor.data_ptr();
- float* out = out_tensor.data_ptr();
-
- gather_points_kernel_launcher_fast(b, c, n, npoints, points, idx, out);
- return 1;
-}
-
-
-int gather_points_grad_wrapper_fast(int b, int c, int n, int npoints,
- at::Tensor grad_out_tensor, at::Tensor idx_tensor, at::Tensor grad_points_tensor) {
-
- CHECK_INPUT(grad_out_tensor);
- CHECK_INPUT(idx_tensor);
- CHECK_INPUT(grad_points_tensor);
-
- const float* grad_out = grad_out_tensor.data_ptr();
- const int* idx = idx_tensor.data_ptr();
- float* grad_points = grad_points_tensor.data_ptr();
-
- gather_points_grad_kernel_launcher_fast(b, c, n, npoints, grad_out, idx, grad_points);
- return 1;
-}
-
-
-int furthest_point_sampling_wrapper(int b, int n, int m,
- at::Tensor points_tensor, at::Tensor temp_tensor, at::Tensor idx_tensor) {
-
- CHECK_INPUT(points_tensor);
- CHECK_INPUT(temp_tensor);
- CHECK_INPUT(idx_tensor);
-
- const float* points = points_tensor.data_ptr();
- float* temp = temp_tensor.data_ptr();
- int* idx = idx_tensor.data_ptr();
-
- furthest_point_sampling_kernel_launcher(b, n, m, points, temp, idx);
- return 1;
-}
diff --git a/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_batch/src/sampling_gpu.cu b/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_batch/src/sampling_gpu.cu
deleted file mode 100644
index 203bd825..00000000
--- a/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_batch/src/sampling_gpu.cu
+++ /dev/null
@@ -1,260 +0,0 @@
-/*
-batch version of point sampling and gathering, modified from the original implementation of official PointNet++ codes.
-Written by Shaoshuai Shi
-All Rights Reserved 2018.
-*/
-
-
-#include
-#include
-
-#include "cuda_utils.h"
-#include "sampling_gpu.h"
-
-
-__global__ void gather_points_kernel_fast(int b, int c, int n, int m,
- const float *__restrict__ points, const int *__restrict__ idx, float *__restrict__ out) {
- // points: (B, C, N)
- // idx: (B, M)
- // output:
- // out: (B, C, M)
-
- int bs_idx = blockIdx.z;
- int c_idx = blockIdx.y;
- int pt_idx = blockIdx.x * blockDim.x + threadIdx.x;
- if (bs_idx >= b || c_idx >= c || pt_idx >= m) return;
-
- out += bs_idx * c * m + c_idx * m + pt_idx;
- idx += bs_idx * m + pt_idx;
- points += bs_idx * c * n + c_idx * n;
- out[0] = points[idx[0]];
-}
-
-void gather_points_kernel_launcher_fast(int b, int c, int n, int npoints,
- const float *points, const int *idx, float *out) {
- // points: (B, C, N)
- // idx: (B, npoints)
- // output:
- // out: (B, C, npoints)
-
- cudaError_t err;
- dim3 blocks(DIVUP(npoints, THREADS_PER_BLOCK), c, b); // blockIdx.x(col), blockIdx.y(row)
- dim3 threads(THREADS_PER_BLOCK);
-
- gather_points_kernel_fast<<>>(b, c, n, npoints, points, idx, out);
-
- err = cudaGetLastError();
- if (cudaSuccess != err) {
- fprintf(stderr, "CUDA kernel failed : %s\n", cudaGetErrorString(err));
- exit(-1);
- }
-}
-
-__global__ void gather_points_grad_kernel_fast(int b, int c, int n, int m, const float *__restrict__ grad_out,
- const int *__restrict__ idx, float *__restrict__ grad_points) {
- // grad_out: (B, C, M)
- // idx: (B, M)
- // output:
- // grad_points: (B, C, N)
-
- int bs_idx = blockIdx.z;
- int c_idx = blockIdx.y;
- int pt_idx = blockIdx.x * blockDim.x + threadIdx.x;
- if (bs_idx >= b || c_idx >= c || pt_idx >= m) return;
-
- grad_out += bs_idx * c * m + c_idx * m + pt_idx;
- idx += bs_idx * m + pt_idx;
- grad_points += bs_idx * c * n + c_idx * n;
-
- atomicAdd(grad_points + idx[0], grad_out[0]);
-}
-
-void gather_points_grad_kernel_launcher_fast(int b, int c, int n, int npoints,
- const float *grad_out, const int *idx, float *grad_points) {
- // grad_out: (B, C, npoints)
- // idx: (B, npoints)
- // output:
- // grad_points: (B, C, N)
-
- cudaError_t err;
- dim3 blocks(DIVUP(npoints, THREADS_PER_BLOCK), c, b); // blockIdx.x(col), blockIdx.y(row)
- dim3 threads(THREADS_PER_BLOCK);
-
- gather_points_grad_kernel_fast<<>>(b, c, n, npoints, grad_out, idx, grad_points);
-
- err = cudaGetLastError();
- if (cudaSuccess != err) {
- fprintf(stderr, "CUDA kernel failed : %s\n", cudaGetErrorString(err));
- exit(-1);
- }
-}
-
-
-__device__ void __update(float *__restrict__ dists, int *__restrict__ dists_i, int idx1, int idx2){
- const float v1 = dists[idx1], v2 = dists[idx2];
- const int i1 = dists_i[idx1], i2 = dists_i[idx2];
- dists[idx1] = max(v1, v2);
- dists_i[idx1] = v2 > v1 ? i2 : i1;
-}
-
-template
-__global__ void furthest_point_sampling_kernel(int b, int n, int m,
- const float *__restrict__ dataset, float *__restrict__ temp, int *__restrict__ idxs) {
- // dataset: (B, N, 3)
- // tmp: (B, N)
- // output:
- // idx: (B, M)
-
- if (m <= 0) return;
- __shared__ float dists[block_size];
- __shared__ int dists_i[block_size];
-
- int batch_index = blockIdx.x;
- dataset += batch_index * n * 3;
- temp += batch_index * n;
- idxs += batch_index * m;
-
- int tid = threadIdx.x;
- const int stride = block_size;
-
- int old = 0;
- if (threadIdx.x == 0)
- idxs[0] = old;
-
- __syncthreads();
- for (int j = 1; j < m; j++) {
- int besti = 0;
- float best = -1;
- float x1 = dataset[old * 3 + 0];
- float y1 = dataset[old * 3 + 1];
- float z1 = dataset[old * 3 + 2];
- for (int k = tid; k < n; k += stride) {
- float x2, y2, z2;
- x2 = dataset[k * 3 + 0];
- y2 = dataset[k * 3 + 1];
- z2 = dataset[k * 3 + 2];
- // float mag = (x2 * x2) + (y2 * y2) + (z2 * z2);
- // if (mag <= 1e-3)
- // continue;
-
- float d = (x2 - x1) * (x2 - x1) + (y2 - y1) * (y2 - y1) + (z2 - z1) * (z2 - z1);
- float d2 = min(d, temp[k]);
- temp[k] = d2;
- besti = d2 > best ? k : besti;
- best = d2 > best ? d2 : best;
- }
- dists[tid] = best;
- dists_i[tid] = besti;
- __syncthreads();
-
- if (block_size >= 1024) {
- if (tid < 512) {
- __update(dists, dists_i, tid, tid + 512);
- }
- __syncthreads();
- }
-
- if (block_size >= 512) {
- if (tid < 256) {
- __update(dists, dists_i, tid, tid + 256);
- }
- __syncthreads();
- }
- if (block_size >= 256) {
- if (tid < 128) {
- __update(dists, dists_i, tid, tid + 128);
- }
- __syncthreads();
- }
- if (block_size >= 128) {
- if (tid < 64) {
- __update(dists, dists_i, tid, tid + 64);
- }
- __syncthreads();
- }
- if (block_size >= 64) {
- if (tid < 32) {
- __update(dists, dists_i, tid, tid + 32);
- }
- __syncthreads();
- }
- if (block_size >= 32) {
- if (tid < 16) {
- __update(dists, dists_i, tid, tid + 16);
- }
- __syncthreads();
- }
- if (block_size >= 16) {
- if (tid < 8) {
- __update(dists, dists_i, tid, tid + 8);
- }
- __syncthreads();
- }
- if (block_size >= 8) {
- if (tid < 4) {
- __update(dists, dists_i, tid, tid + 4);
- }
- __syncthreads();
- }
- if (block_size >= 4) {
- if (tid < 2) {
- __update(dists, dists_i, tid, tid + 2);
- }
- __syncthreads();
- }
- if (block_size >= 2) {
- if (tid < 1) {
- __update(dists, dists_i, tid, tid + 1);
- }
- __syncthreads();
- }
-
- old = dists_i[0];
- if (tid == 0)
- idxs[j] = old;
- }
-}
-
-void furthest_point_sampling_kernel_launcher(int b, int n, int m,
- const float *dataset, float *temp, int *idxs) {
- // dataset: (B, N, 3)
- // tmp: (B, N)
- // output:
- // idx: (B, M)
-
- cudaError_t err;
- unsigned int n_threads = opt_n_threads(n);
-
- switch (n_threads) {
- case 1024:
- furthest_point_sampling_kernel<1024><<>>(b, n, m, dataset, temp, idxs); break;
- case 512:
- furthest_point_sampling_kernel<512><<>>(b, n, m, dataset, temp, idxs); break;
- case 256:
- furthest_point_sampling_kernel<256><<>>(b, n, m, dataset, temp, idxs); break;
- case 128:
- furthest_point_sampling_kernel<128><<>>(b, n, m, dataset, temp, idxs); break;
- case 64:
- furthest_point_sampling_kernel<64><<>>(b, n, m, dataset, temp, idxs); break;
- case 32:
- furthest_point_sampling_kernel<32><<>>(b, n, m, dataset, temp, idxs); break;
- case 16:
- furthest_point_sampling_kernel<16><<>>(b, n, m, dataset, temp, idxs); break;
- case 8:
- furthest_point_sampling_kernel<8><<>>(b, n, m, dataset, temp, idxs); break;
- case 4:
- furthest_point_sampling_kernel<4><<>>(b, n, m, dataset, temp, idxs); break;
- case 2:
- furthest_point_sampling_kernel<2><<>>(b, n, m, dataset, temp, idxs); break;
- case 1:
- furthest_point_sampling_kernel<1><<>>(b, n, m, dataset, temp, idxs); break;
- default:
- furthest_point_sampling_kernel<512><<>>(b, n, m, dataset, temp, idxs);
- }
-
- err = cudaGetLastError();
- if (cudaSuccess != err) {
- fprintf(stderr, "CUDA kernel failed : %s\n", cudaGetErrorString(err));
- exit(-1);
- }
-}
diff --git a/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_batch/src/sampling_gpu.h b/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_batch/src/sampling_gpu.h
deleted file mode 100644
index 034e04cb..00000000
--- a/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_batch/src/sampling_gpu.h
+++ /dev/null
@@ -1,29 +0,0 @@
-#ifndef _SAMPLING_GPU_H
-#define _SAMPLING_GPU_H
-
-#include
-#include
-#include
-
-
-int gather_points_wrapper_fast(int b, int c, int n, int npoints,
- at::Tensor points_tensor, at::Tensor idx_tensor, at::Tensor out_tensor);
-
-void gather_points_kernel_launcher_fast(int b, int c, int n, int npoints,
- const float *points, const int *idx, float *out);
-
-
-int gather_points_grad_wrapper_fast(int b, int c, int n, int npoints,
- at::Tensor grad_out_tensor, at::Tensor idx_tensor, at::Tensor grad_points_tensor);
-
-void gather_points_grad_kernel_launcher_fast(int b, int c, int n, int npoints,
- const float *grad_out, const int *idx, float *grad_points);
-
-
-int furthest_point_sampling_wrapper(int b, int n, int m,
- at::Tensor points_tensor, at::Tensor temp_tensor, at::Tensor idx_tensor);
-
-void furthest_point_sampling_kernel_launcher(int b, int n, int m,
- const float *dataset, float *temp, int *idxs);
-
-#endif
diff --git a/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_stack/pointnet2_modules.py b/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_stack/pointnet2_modules.py
deleted file mode 100644
index fa3d450d..00000000
--- a/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_stack/pointnet2_modules.py
+++ /dev/null
@@ -1,80 +0,0 @@
-from typing import List
-
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-
-from opencood.pcdet_utils.pointnet2.pointnet2_stack import pointnet2_utils
-
-
-class StackSAModuleMSG(nn.Module):
- def __init__(self, *, radii: List[float], nsamples: List[int], mlps: List[List[int]], use_xyz: bool = True, pool_method="max_pool"):
- """
- Args:
- radii: list of float, list of radii to group with
- nsamples: list of int, number of samples in each ball query
- mlps: list of list of int, spec of the pointnet before the global pooling for each scale
- use_xyz:
- pool_method: max_pool / avg_pool
- """
- super().__init__()
-
- assert len(radii) == len(nsamples) == len(mlps)
-
- self.groupers = nn.ModuleList()
- self.mlps = nn.ModuleList()
- for i in range(len(radii)):
- radius = radii[i]
- nsample = nsamples[i]
- self.groupers.append(pointnet2_utils.QueryAndGroup(radius, nsample, use_xyz=use_xyz))
- mlp_spec = mlps[i]
- if use_xyz:
- mlp_spec[0] += 3
-
- shared_mlps = []
- for k in range(len(mlp_spec) - 1):
- shared_mlps.extend([nn.Conv2d(mlp_spec[k], mlp_spec[k + 1], kernel_size=1, bias=False), nn.BatchNorm2d(mlp_spec[k + 1]), nn.ReLU()])
- self.mlps.append(nn.Sequential(*shared_mlps))
- self.pool_method = pool_method
-
- self.init_weights()
-
- def init_weights(self):
- for m in self.modules():
- if isinstance(m, nn.Conv2d):
- nn.init.kaiming_normal_(m.weight)
- if m.bias is not None:
- nn.init.constant_(m.bias, 0)
- if isinstance(m, nn.BatchNorm2d):
- nn.init.constant_(m.weight, 1.0)
- nn.init.constant_(m.bias, 0)
-
- def forward(self, xyz, xyz_batch_cnt, new_xyz, new_xyz_batch_cnt, features=None, empty_voxel_set_zeros=True):
- r"""
- :param xyz: (N1 + N2 ..., 3) tensor of the xyz coordinates of the features
- :param xyz_batch_cnt: (batch_size), [N1, N2, ...]
- :param new_xyz: (M1 + M2 ..., 3)
- :param new_xyz_batch_cnt: (batch_size), [M1, M2, ...]
- :param features: (N1 + N2 ..., C) tensor of the descriptors of the the features
- :return:
- new_xyz: (M1 + M2 ..., 3) tensor of the new features' xyz
- new_features: (M1 + M2 ..., \sum_k(mlps[k][-1])) tensor of the new_features descriptors
- """
- new_features_list = []
- for k in range(len(self.groupers)):
- new_features, _ = self.groupers[k](xyz, xyz_batch_cnt, new_xyz, new_xyz_batch_cnt, features) # (M1 + M2, C, nsample)
- new_features = new_features.permute(1, 0, 2).unsqueeze(dim=0) # (1, C, M1 + M2 ..., nsample)
- new_features = self.mlps[k](new_features) # (1, C, M1 + M2 ..., nsample)
-
- if self.pool_method == "max_pool":
- new_features = F.max_pool2d(new_features, kernel_size=[1, new_features.size(3)]).squeeze(dim=-1) # (1, C, M1 + M2 ...)
- elif self.pool_method == "avg_pool":
- new_features = F.avg_pool2d(new_features, kernel_size=[1, new_features.size(3)]).squeeze(dim=-1) # (1, C, M1 + M2 ...)
- else:
- raise NotImplementedError
- new_features = new_features.squeeze(dim=0).permute(1, 0) # (M1 + M2 ..., C)
- new_features_list.append(new_features)
-
- new_features = torch.cat(new_features_list, dim=1) # (M1 + M2 ..., C)
-
- return new_xyz, new_features
diff --git a/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_stack/pointnet2_utils.py b/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_stack/pointnet2_utils.py
deleted file mode 100644
index 637b2f71..00000000
--- a/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_stack/pointnet2_utils.py
+++ /dev/null
@@ -1,176 +0,0 @@
-import torch
-import torch.nn as nn
-from torch.autograd import Function, Variable
-
-from opencood.pcdet_utils.pointnet2.pointnet2_stack import pointnet2_stack_cuda as pointnet2
-
-
-class BallQuery(Function):
- @staticmethod
- def forward(ctx, radius: float, nsample: int, xyz: torch.Tensor, xyz_batch_cnt: torch.Tensor, new_xyz: torch.Tensor, new_xyz_batch_cnt):
- """
- Args:
- ctx:
- radius: float, radius of the balls
- nsample: int, maximum number of features in the balls
- xyz: (N1 + N2 ..., 3) xyz coordinates of the features
- xyz_batch_cnt: (batch_size), [N1, N2, ...]
- new_xyz: (M1 + M2 ..., 3) centers of the ball query
- new_xyz_batch_cnt: (batch_size), [M1, M2, ...]
-
- Returns:
- idx: (M1 + M2, nsample) tensor with the indicies of the features that form the query balls
- """
- assert new_xyz.is_contiguous()
- assert new_xyz_batch_cnt.is_contiguous()
- assert xyz.is_contiguous()
- assert xyz_batch_cnt.is_contiguous()
-
- B = xyz_batch_cnt.shape[0]
- M = new_xyz.shape[0]
- idx = torch.cuda.IntTensor(M, nsample).zero_()
-
- pointnet2.ball_query_wrapper(B, M, radius, nsample, new_xyz, new_xyz_batch_cnt, xyz, xyz_batch_cnt, idx)
- empty_ball_mask = idx[:, 0] == -1
- idx[empty_ball_mask] = 0
- return idx, empty_ball_mask
-
- @staticmethod
- def backward(ctx, a=None):
- return None, None, None, None
-
-
-ball_query = BallQuery.apply
-
-
-class GroupingOperation(Function):
- @staticmethod
- def forward(ctx, features: torch.Tensor, features_batch_cnt: torch.Tensor, idx: torch.Tensor, idx_batch_cnt: torch.Tensor):
- """
- Args:
- ctx:
- features: (N1 + N2 ..., C) tensor of features to group
- features_batch_cnt: (batch_size) [N1 + N2 ...] tensor containing the indicies of features to group with
- idx: (M1 + M2 ..., nsample) tensor containing the indicies of features to group with
- idx_batch_cnt: (batch_size) [M1 + M2 ...] tensor containing the indicies of features to group with
-
- Returns:
- output: (M1 + M2, C, nsample) tensor
- """
- assert features.is_contiguous()
- assert features_batch_cnt.is_contiguous()
- assert idx.is_contiguous()
- assert idx_batch_cnt.is_contiguous()
-
- assert features.shape[0] == features_batch_cnt.sum(), "features: %s, features_batch_cnt: %s" % (str(features.shape), str(features_batch_cnt))
- assert idx.shape[0] == idx_batch_cnt.sum(), "idx: %s, idx_batch_cnt: %s" % (str(idx.shape), str(idx_batch_cnt))
-
- M, nsample = idx.size()
- N, C = features.size()
- B = idx_batch_cnt.shape[0]
- output = torch.cuda.FloatTensor(M, C, nsample)
-
- pointnet2.group_points_wrapper(B, M, C, nsample, features, features_batch_cnt, idx, idx_batch_cnt, output)
-
- ctx.for_backwards = (B, N, idx, features_batch_cnt, idx_batch_cnt)
- return output
-
- @staticmethod
- def backward(ctx, grad_out: torch.Tensor):
- """
- Args:
- ctx:
- grad_out: (M1 + M2 ..., C, nsample) tensor of the gradients of the output from forward
-
- Returns:
- grad_features: (N1 + N2 ..., C) gradient of the features
- """
- B, N, idx, features_batch_cnt, idx_batch_cnt = ctx.for_backwards
-
- M, C, nsample = grad_out.size()
- grad_features = Variable(torch.cuda.FloatTensor(N, C).zero_())
-
- grad_out_data = grad_out.data.contiguous()
- pointnet2.group_points_grad_wrapper(B, M, C, N, nsample, grad_out_data, idx, idx_batch_cnt, features_batch_cnt, grad_features.data)
- return grad_features, None, None, None
-
-
-grouping_operation = GroupingOperation.apply
-
-
-class QueryAndGroup(nn.Module):
- def __init__(self, radius: float, nsample: int, use_xyz: bool = True):
- """
- Args:
- radius: float, radius of ball
- nsample: int, maximum number of features to gather in the ball
- use_xyz:
- """
- super().__init__()
- self.radius, self.nsample, self.use_xyz = radius, nsample, use_xyz
-
- def forward(
- self, xyz: torch.Tensor, xyz_batch_cnt: torch.Tensor, new_xyz: torch.Tensor, new_xyz_batch_cnt: torch.Tensor, features: torch.Tensor = None
- ):
- """
- Args:
- xyz: (N1 + N2 ..., 3) xyz coordinates of the features
- xyz_batch_cnt: (batch_size), [N1, N2, ...]
- new_xyz: (M1 + M2 ..., 3) centers of the ball query
- new_xyz_batch_cnt: (batch_size), [M1, M2, ...]
- features: (N1 + N2 ..., C) tensor of features to group
-
- Returns:
- new_features: (M1 + M2, C, nsample) tensor
- """
- assert xyz.shape[0] == xyz_batch_cnt.sum(), "xyz: %s, xyz_batch_cnt: %s" % (str(xyz.shape), str(new_xyz_batch_cnt))
- assert new_xyz.shape[0] == new_xyz_batch_cnt.sum(), "new_xyz: %s, new_xyz_batch_cnt: %s" % (str(new_xyz.shape), str(new_xyz_batch_cnt))
-
- # idx: (M1 + M2 ..., nsample), empty_ball_mask: (M1 + M2 ...)
- idx, empty_ball_mask = ball_query(self.radius, self.nsample, xyz, xyz_batch_cnt, new_xyz, new_xyz_batch_cnt)
- grouped_xyz = grouping_operation(xyz, xyz_batch_cnt, idx, new_xyz_batch_cnt) # (M1 + M2, 3, nsample)
- grouped_xyz -= new_xyz.unsqueeze(-1)
-
- grouped_xyz[empty_ball_mask] = 0
-
- if features is not None:
- grouped_features = grouping_operation(features, xyz_batch_cnt, idx, new_xyz_batch_cnt) # (M1 + M2, C, nsample)
- grouped_features[empty_ball_mask] = 0
- if self.use_xyz:
- new_features = torch.cat([grouped_xyz, grouped_features], dim=1) # (M1 + M2 ..., C + 3, nsample)
- else:
- new_features = grouped_features
- else:
- assert self.use_xyz, "Cannot have not features and not use xyz as a feature!"
- new_features = grouped_xyz
-
- return new_features, idx
-
-
-class FurthestPointSampling(Function):
- @staticmethod
- def forward(ctx, xyz: torch.Tensor, npoint: int):
- """
- Args:
- ctx:
- xyz: (B, N, 3) where N > npoint
- npoint: int, number of features in the sampled set
-
- Returns:
- output: (B, npoint) tensor containing the set
- """
- assert xyz.is_contiguous()
-
- B, N, _ = xyz.size()
- output = torch.cuda.IntTensor(B, npoint)
- temp = torch.cuda.FloatTensor(B, N).fill_(1e10)
-
- pointnet2.furthest_point_sampling_wrapper(B, N, npoint, xyz, temp, output)
- return output
-
- @staticmethod
- def backward(xyz, a=None):
- return None, None
-
-
-furthest_point_sample = FurthestPointSampling.apply
diff --git a/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_stack/src/ball_query.cpp b/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_stack/src/ball_query.cpp
deleted file mode 100644
index 2fcb5baa..00000000
--- a/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_stack/src/ball_query.cpp
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
-Stacked-batch-data version of ball query, modified from the original implementation of official PointNet++ codes.
-Written by Shaoshuai Shi
-All Rights Reserved 2019-2020.
-*/
-
-#include
-#include
-#include
-#include
-#include "ball_query_gpu.h"
-
-#define CHECK_CUDA(x) TORCH_CHECK(x.is_cuda(), #x " must be a CUDA tensor")
-#define CHECK_CONTIGUOUS(x) TORCH_CHECK(x.is_contiguous(), #x " must be contiguous")
-#define CHECK_INPUT(x) CHECK_CUDA(x); CHECK_CONTIGUOUS(x)
-
-int ball_query_wrapper_stack(int B, int M, float radius, int nsample,
- at::Tensor new_xyz_tensor, at::Tensor new_xyz_batch_cnt_tensor,
- at::Tensor xyz_tensor, at::Tensor xyz_batch_cnt_tensor, at::Tensor idx_tensor) {
-
- CHECK_INPUT(new_xyz_tensor);
- CHECK_INPUT(xyz_tensor);
- CHECK_INPUT(new_xyz_batch_cnt_tensor);
- CHECK_INPUT(xyz_batch_cnt_tensor);
- CHECK_INPUT(idx_tensor);
-
- const float* new_xyz = new_xyz_tensor.data_ptr();
- const float* xyz = xyz_tensor.data_ptr();
- const int* new_xyz_batch_cnt = new_xyz_batch_cnt_tensor.data_ptr();
- const int* xyz_batch_cnt = xyz_batch_cnt_tensor.data_ptr();
- int* idx = idx_tensor.data_ptr();
-
- ball_query_kernel_launcher_stack(B, M, radius, nsample, new_xyz, new_xyz_batch_cnt, xyz, xyz_batch_cnt, idx);
- return 1;
-}
diff --git a/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_stack/src/ball_query_gpu.cu b/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_stack/src/ball_query_gpu.cu
deleted file mode 100644
index adaa6b1e..00000000
--- a/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_stack/src/ball_query_gpu.cu
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
-Stacked-batch-data version of ball query, modified from the original implementation of official PointNet++ codes.
-Written by Shaoshuai Shi
-All Rights Reserved 2019-2020.
-*/
-
-
-#include
-#include
-#include
-
-#include "ball_query_gpu.h"
-#include "cuda_utils.h"
-
-
-__global__ void ball_query_kernel_stack(int B, int M, float radius, int nsample, \
- const float *new_xyz, const int *new_xyz_batch_cnt, const float *xyz, const int *xyz_batch_cnt, int *idx) {
- // :param xyz: (N1 + N2 ..., 3) xyz coordinates of the features
- // :param xyz_batch_cnt: (batch_size), [N1, N2, ...]
- // :param new_xyz: (M1 + M2 ..., 3) centers of the ball query
- // :param new_xyz_batch_cnt: (batch_size), [M1, M2, ...]
- // output:
- // idx: (M, nsample)
- int pt_idx = blockIdx.x * blockDim.x + threadIdx.x;
- if (pt_idx >= M) return;
-
- int bs_idx = 0, pt_cnt = new_xyz_batch_cnt[0];
- for (int k = 1; k < B; k++){
- if (pt_idx < pt_cnt) break;
- pt_cnt += new_xyz_batch_cnt[k];
- bs_idx = k;
- }
-
- int xyz_batch_start_idx = 0;
- for (int k = 0; k < bs_idx; k++) xyz_batch_start_idx += xyz_batch_cnt[k];
- // for (int k = 0; k < bs_idx; k++) new_xyz_batch_start_idx += new_xyz_batch_cnt[k];
-
- new_xyz += pt_idx * 3;
- xyz += xyz_batch_start_idx * 3;
- idx += pt_idx * nsample;
-
- float radius2 = radius * radius;
- float new_x = new_xyz[0];
- float new_y = new_xyz[1];
- float new_z = new_xyz[2];
- int n = xyz_batch_cnt[bs_idx];
-
- int cnt = 0;
- for (int k = 0; k < n; ++k) {
- float x = xyz[k * 3 + 0];
- float y = xyz[k * 3 + 1];
- float z = xyz[k * 3 + 2];
- float d2 = (new_x - x) * (new_x - x) + (new_y - y) * (new_y - y) + (new_z - z) * (new_z - z);
- if (d2 < radius2){
- if (cnt == 0){
- for (int l = 0; l < nsample; ++l) {
- idx[l] = k;
- }
- }
- idx[cnt] = k;
- ++cnt;
- if (cnt >= nsample) break;
- }
- }
- if (cnt == 0) idx[0] = -1;
-}
-
-
-void ball_query_kernel_launcher_stack(int B, int M, float radius, int nsample,
- const float *new_xyz, const int *new_xyz_batch_cnt, const float *xyz, const int *xyz_batch_cnt, int *idx){
- // :param xyz: (N1 + N2 ..., 3) xyz coordinates of the features
- // :param xyz_batch_cnt: (batch_size), [N1, N2, ...]
- // :param new_xyz: (M1 + M2 ..., 3) centers of the ball query
- // :param new_xyz_batch_cnt: (batch_size), [M1, M2, ...]
- // output:
- // idx: (M, nsample)
-
- cudaError_t err;
-
- dim3 blocks(DIVUP(M, THREADS_PER_BLOCK)); // blockIdx.x(col), blockIdx.y(row)
- dim3 threads(THREADS_PER_BLOCK);
-
- ball_query_kernel_stack<<>>(B, M, radius, nsample, new_xyz, new_xyz_batch_cnt, xyz, xyz_batch_cnt, idx);
- // cudaDeviceSynchronize(); // for using printf in kernel function
- err = cudaGetLastError();
- if (cudaSuccess != err) {
- fprintf(stderr, "CUDA kernel failed : %s\n", cudaGetErrorString(err));
- exit(-1);
- }
-}
diff --git a/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_stack/src/ball_query_gpu.h b/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_stack/src/ball_query_gpu.h
deleted file mode 100644
index c74f1201..00000000
--- a/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_stack/src/ball_query_gpu.h
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
-Stacked-batch-data version of ball query, modified from the original implementation of official PointNet++ codes.
-Written by Shaoshuai Shi
-All Rights Reserved 2019-2020.
-*/
-
-
-#ifndef _STACK_BALL_QUERY_GPU_H
-#define _STACK_BALL_QUERY_GPU_H
-
-#include
-#include
-#include
-#include
-
-int ball_query_wrapper_stack(int B, int M, float radius, int nsample,
- at::Tensor new_xyz_tensor, at::Tensor new_xyz_batch_cnt_tensor,
- at::Tensor xyz_tensor, at::Tensor xyz_batch_cnt_tensor, at::Tensor idx_tensor);
-
-
-void ball_query_kernel_launcher_stack(int B, int M, float radius, int nsample,
- const float *new_xyz, const int *new_xyz_batch_cnt, const float *xyz, const int *xyz_batch_cnt, int *idx);
-
-
-#endif
diff --git a/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_stack/src/cuda_utils.h b/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_stack/src/cuda_utils.h
deleted file mode 100644
index c1670f1c..00000000
--- a/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_stack/src/cuda_utils.h
+++ /dev/null
@@ -1,9 +0,0 @@
-#ifndef _STACK_CUDA_UTILS_H
-#define _STACK_CUDA_UTILS_H
-
-#include
-
-#define THREADS_PER_BLOCK 256
-#define DIVUP(m,n) ((m) / (n) + ((m) % (n) > 0))
-
-#endif
diff --git a/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_stack/src/group_points.cpp b/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_stack/src/group_points.cpp
deleted file mode 100644
index d6ebd0dc..00000000
--- a/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_stack/src/group_points.cpp
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
-Stacked-batch-data version of point grouping, modified from the original implementation of official PointNet++ codes.
-Written by Shaoshuai Shi
-All Rights Reserved 2019-2020.
-*/
-
-#include
-#include
-#include
-#include
-
-#include "group_points_gpu.h"
-
-#define CHECK_CUDA(x) TORCH_CHECK(x.is_cuda(), #x " must be a CUDA tensor")
-#define CHECK_CONTIGUOUS(x) TORCH_CHECK(x.is_contiguous(), #x " must be contiguous")
-#define CHECK_INPUT(x) CHECK_CUDA(x); CHECK_CONTIGUOUS(x)
-
-
-int group_points_grad_wrapper_stack(int B, int M, int C, int N, int nsample,
- at::Tensor grad_out_tensor, at::Tensor idx_tensor, at::Tensor idx_batch_cnt_tensor,
- at::Tensor features_batch_cnt_tensor, at::Tensor grad_features_tensor) {
-
- CHECK_INPUT(grad_out_tensor);
- CHECK_INPUT(idx_tensor);
- CHECK_INPUT(idx_batch_cnt_tensor);
- CHECK_INPUT(features_batch_cnt_tensor);
- CHECK_INPUT(grad_features_tensor);
-
- const float* grad_out = grad_out_tensor.data_ptr();
- const int* idx = idx_tensor.data_ptr();
- const int* idx_batch_cnt = idx_batch_cnt_tensor.data_ptr();
- const int* features_batch_cnt = features_batch_cnt_tensor.data_ptr();
- float* grad_features = grad_features_tensor.data_ptr();
-
- group_points_grad_kernel_launcher_stack(B, M, C, N, nsample, grad_out, idx, idx_batch_cnt, features_batch_cnt, grad_features);
- return 1;
-}
-
-
-int group_points_wrapper_stack(int B, int M, int C, int nsample,
- at::Tensor features_tensor, at::Tensor features_batch_cnt_tensor,
- at::Tensor idx_tensor, at::Tensor idx_batch_cnt_tensor, at::Tensor out_tensor) {
-
- CHECK_INPUT(features_tensor);
- CHECK_INPUT(features_batch_cnt_tensor);
- CHECK_INPUT(idx_tensor);
- CHECK_INPUT(idx_batch_cnt_tensor);
- CHECK_INPUT(out_tensor);
-
- const float* features = features_tensor.data_ptr();
- const int* idx = idx_tensor.data_ptr();
- const int* features_batch_cnt = features_batch_cnt_tensor.data_ptr();
- const int* idx_batch_cnt = idx_batch_cnt_tensor.data_ptr();
- float* out = out_tensor.data_ptr();
-
- group_points_kernel_launcher_stack(B, M, C, nsample, features, features_batch_cnt, idx, idx_batch_cnt, out);
- return 1;
-}
diff --git a/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_stack/src/group_points_gpu.cu b/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_stack/src/group_points_gpu.cu
deleted file mode 100644
index 62e341e9..00000000
--- a/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_stack/src/group_points_gpu.cu
+++ /dev/null
@@ -1,125 +0,0 @@
-/*
-Stacked-batch-data version of point grouping, modified from the original implementation of official PointNet++ codes.
-Written by Shaoshuai Shi
-All Rights Reserved 2019-2020.
-*/
-
-
-#include
-#include
-
-#include "cuda_utils.h"
-#include "group_points_gpu.h"
-
-
-__global__ void group_points_grad_kernel_stack(int B, int M, int C, int N, int nsample,
- const float *grad_out, const int *idx, const int *idx_batch_cnt, const int *features_batch_cnt, float *grad_features) {
- // :param grad_out: (M1 + M2 ..., C, nsample) tensor of the gradients of the output from forward
- // :param idx: (M1 + M2 ..., nsample) tensor containing the indicies of features to group with
- // :param idx_batch_cnt: (batch_size) [M1 + M2 ...] tensor containing the indicies of features to group with
- // :param features_batch_cnt: (batch_size) [N1 + N2 ...] tensor containing the indicies of features to group with
- // :return:
- // grad_features: (N1 + N2 ..., C) gradient of the features
- int index = blockIdx.x * blockDim.x + threadIdx.x;
- int sample_idx = index % nsample;
- int C_idx = (index / nsample) % C;
- int pt_idx = (index / nsample / C);
-
- if (pt_idx >= M || C_idx >= C || sample_idx >= nsample) return;
-
- int bs_idx = 0, pt_cnt = idx_batch_cnt[0];
- for (int k = 1; k < B; k++){
- if (pt_idx < pt_cnt) break;
- pt_cnt += idx_batch_cnt[k];
- bs_idx = k;
- }
-
- int features_batch_start_idx = 0;
- for (int k = 0; k < bs_idx; k++) features_batch_start_idx += features_batch_cnt[k];
-
- grad_out += pt_idx * C * nsample + C_idx * nsample + sample_idx;
- idx += pt_idx * nsample + sample_idx;
- grad_features += (features_batch_start_idx + idx[0]) * C + C_idx;
-
- atomicAdd(grad_features, grad_out[0]);
-}
-
-void group_points_grad_kernel_launcher_stack(int B, int M, int C, int N, int nsample,
- const float *grad_out, const int *idx, const int *idx_batch_cnt, const int *features_batch_cnt, float *grad_features) {
- // :param grad_out: (M1 + M2 ..., C, nsample) tensor of the gradients of the output from forward
- // :param idx: (M1 + M2 ..., nsample) tensor containing the indicies of features to group with
- // :param idx_batch_cnt: (batch_size) [M1 + M2 ...] tensor containing the indicies of features to group with
- // :param features_batch_cnt: (batch_size) [N1 + N2 ...] tensor containing the indicies of features to group with
- // :return:
- // grad_features: (N1 + N2 ..., C) gradient of the features
-
- cudaError_t err;
- // dim3 blocks(DIVUP(npoints * nsample, THREADS_PER_BLOCK), c, b); // blockIdx.x(col), blockIdx.y(row)
- dim3 blocks(DIVUP(M * C * nsample, THREADS_PER_BLOCK)); // blockIdx.x(col), blockIdx.y(row)
- dim3 threads(THREADS_PER_BLOCK);
-
- group_points_grad_kernel_stack<<>>(B, M, C, N, nsample, grad_out, idx, idx_batch_cnt, features_batch_cnt, grad_features);
-
- err = cudaGetLastError();
- if (cudaSuccess != err) {
- fprintf(stderr, "CUDA kernel failed : %s\n", cudaGetErrorString(err));
- exit(-1);
- }
-}
-
-
-__global__ void group_points_kernel_stack(int B, int M, int C, int nsample,
- const float *features, const int *features_batch_cnt, const int *idx, const int *idx_batch_cnt, float *out) {
- // :param features: (N1 + N2 ..., C) tensor of features to group
- // :param features_batch_cnt: (batch_size) [N1 + N2 ...] tensor containing the indicies of features to group with
- // :param idx: (M1 + M2 ..., nsample) tensor containing the indicies of features to group with
- // :param idx_batch_cnt: (batch_size) [M1 + M2 ...] tensor containing the indicies of features to group with
- // :return:
- // output: (M1 + M2, C, nsample) tensor
- int index = blockIdx.x * blockDim.x + threadIdx.x;
- int sample_idx = index % nsample;
- int C_idx = (index / nsample) % C;
- int pt_idx = (index / nsample / C);
-
- if (pt_idx >= M || C_idx >= C || sample_idx >= nsample) return;
-
- int bs_idx = 0, pt_cnt = idx_batch_cnt[0];
- for (int k = 1; k < B; k++){
- if (pt_idx < pt_cnt) break;
- pt_cnt += idx_batch_cnt[k];
- bs_idx = k;
- }
-
- int features_batch_start_idx = 0;
- for (int k = 0; k < bs_idx; k++) features_batch_start_idx += features_batch_cnt[k];
- features += features_batch_start_idx * C;
-
- idx += pt_idx * nsample + sample_idx;
- int in_idx = idx[0] * C + C_idx;
- int out_idx = pt_idx * C * nsample + C_idx * nsample + sample_idx;
-
- out[out_idx] = features[in_idx];
-}
-
-
-void group_points_kernel_launcher_stack(int B, int M, int C, int nsample,
- const float *features, const int *features_batch_cnt, const int *idx, const int *idx_batch_cnt, float *out) {
- // :param features: (N1 + N2 ..., C) tensor of features to group
- // :param features_batch_cnt: (batch_size) [N1 + N2 ...] tensor containing the indicies of features to group with
- // :param idx: (M1 + M2 ..., nsample) tensor containing the indicies of features to group with
- // :param idx_batch_cnt: (batch_size) [M1 + M2 ...] tensor containing the indicies of features to group with
- // :return:
- // output: (M1 + M2, C, nsample) tensor
-
- cudaError_t err;
- dim3 blocks(DIVUP(M * C * nsample, THREADS_PER_BLOCK)); // blockIdx.x(col), blockIdx.y(row)
- dim3 threads(THREADS_PER_BLOCK);
-
- group_points_kernel_stack<<>>(B, M, C, nsample, features, features_batch_cnt, idx, idx_batch_cnt, out);
- // cudaDeviceSynchronize(); // for using printf in kernel function
- err = cudaGetLastError();
- if (cudaSuccess != err) {
- fprintf(stderr, "CUDA kernel failed : %s\n", cudaGetErrorString(err));
- exit(-1);
- }
-}
diff --git a/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_stack/src/group_points_gpu.h b/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_stack/src/group_points_gpu.h
deleted file mode 100644
index 4a266216..00000000
--- a/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_stack/src/group_points_gpu.h
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
-Stacked-batch-data version of point grouping, modified from the original implementation of official PointNet++ codes.
-Written by Shaoshuai Shi
-All Rights Reserved 2019-2020.
-*/
-
-
-#ifndef _STACK_GROUP_POINTS_GPU_H
-#define _STACK_GROUP_POINTS_GPU_H
-
-#include
-#include
-#include
-#include
-
-
-int group_points_wrapper_stack(int B, int M, int C, int nsample,
- at::Tensor features_tensor, at::Tensor features_batch_cnt_tensor,
- at::Tensor idx_tensor, at::Tensor idx_batch_cnt_tensor, at::Tensor out_tensor);
-
-void group_points_kernel_launcher_stack(int B, int M, int C, int nsample,
- const float *features, const int *features_batch_cnt, const int *idx, const int *idx_batch_cnt, float *out);
-
-int group_points_grad_wrapper_stack(int B, int M, int C, int N, int nsample,
- at::Tensor grad_out_tensor, at::Tensor idx_tensor, at::Tensor idx_batch_cnt_tensor,
- at::Tensor features_batch_cnt_tensor, at::Tensor grad_features_tensor);
-
-void group_points_grad_kernel_launcher_stack(int B, int M, int C, int N, int nsample,
- const float *grad_out, const int *idx, const int *idx_batch_cnt, const int *features_batch_cnt, float *grad_features);
-
-#endif
diff --git a/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_stack/src/interpolate.cpp b/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_stack/src/interpolate.cpp
deleted file mode 100644
index faf78baf..00000000
--- a/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_stack/src/interpolate.cpp
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
-Stacked-batch-data version of point interpolation, modified from the original implementation of official PointNet++ codes.
-Written by Shaoshuai Shi
-All Rights Reserved 2019-2020.
-*/
-
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include "interpolate_gpu.h"
-
-#define CHECK_CUDA(x) TORCH_CHECK(x.is_cuda(), #x " must be a CUDA tensor")
-#define CHECK_CONTIGUOUS(x) TORCH_CHECK(x.is_contiguous(), #x " must be contiguous")
-#define CHECK_INPUT(x) CHECK_CUDA(x); CHECK_CONTIGUOUS(x)
-
-
-void three_nn_wrapper_stack(at::Tensor unknown_tensor,
- at::Tensor unknown_batch_cnt_tensor, at::Tensor known_tensor,
- at::Tensor known_batch_cnt_tensor, at::Tensor dist2_tensor, at::Tensor idx_tensor) {
-
- CHECK_INPUT(unknown_tensor);
- CHECK_INPUT(unknown_batch_cnt_tensor);
- CHECK_INPUT(known_tensor);
- CHECK_INPUT(known_batch_cnt_tensor);
- CHECK_INPUT(dist2_tensor);
- CHECK_INPUT(idx_tensor);
-
- int batch_size = unknown_batch_cnt_tensor.size(0);
- int N = unknown_tensor.size(0);
- int M = known_tensor.size(0);
-
- const float* unknown = unknown_tensor.data_ptr();
- const int* unknown_batch_cnt = unknown_batch_cnt_tensor.data_ptr();
- const float* known = known_tensor.data_ptr();
- const int* known_batch_cnt = known_batch_cnt_tensor.data_ptr();
- float* dist2 = dist2_tensor.data_ptr();
- int* idx = idx_tensor.data_ptr();
-
- three_nn_kernel_launcher_stack(batch_size, N, M, unknown, unknown_batch_cnt, known, known_batch_cnt, dist2, idx);
-}
-
-
-void three_interpolate_wrapper_stack(at::Tensor features_tensor,
- at::Tensor idx_tensor, at::Tensor weight_tensor, at::Tensor out_tensor) {
-
- CHECK_INPUT(features_tensor);
- CHECK_INPUT(idx_tensor);
- CHECK_INPUT(weight_tensor);
- CHECK_INPUT(out_tensor);
-
- int N = out_tensor.size(0);
- int channels = features_tensor.size(1);
-
- const float* features = features_tensor.data_ptr();
- const float* weight = weight_tensor.data_ptr();
- const int* idx = idx_tensor.data_ptr();
- float* out = out_tensor.data_ptr();
-
- three_interpolate_kernel_launcher_stack(N, channels, features, idx, weight, out);
-}
-
-
-void three_interpolate_grad_wrapper_stack(at::Tensor grad_out_tensor, at::Tensor idx_tensor,
- at::Tensor weight_tensor, at::Tensor grad_features_tensor) {
-
- CHECK_INPUT(grad_out_tensor);
- CHECK_INPUT(idx_tensor);
- CHECK_INPUT(weight_tensor);
- CHECK_INPUT(grad_features_tensor);
-
- int N = grad_out_tensor.size(0);
- int channels = grad_out_tensor.size(1);
-
- const float* grad_out = grad_out_tensor.data_ptr();
- const float* weight = weight_tensor.data_ptr();
- const int* idx = idx_tensor.data_ptr();
- float* grad_features = grad_features_tensor.data_ptr();
-
- three_interpolate_grad_kernel_launcher_stack(N, channels, grad_out, idx, weight, grad_features);
-}
diff --git a/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_stack/src/interpolate_gpu.cu b/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_stack/src/interpolate_gpu.cu
deleted file mode 100644
index 3a0eb4e8..00000000
--- a/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_stack/src/interpolate_gpu.cu
+++ /dev/null
@@ -1,195 +0,0 @@
-/*
-Stacked-batch-data version of point interpolation, modified from the original implementation of official PointNet++ codes.
-Written by Shaoshuai Shi
-All Rights Reserved 2019-2020.
-*/
-
-
-#include
-#include
-#include
-
-#include "cuda_utils.h"
-#include "interpolate_gpu.h"
-
-
-__global__ void three_nn_kernel_stack(int batch_size, int N, int M, const float *unknown,
- const int *unknown_batch_cnt, const float *known, const int *known_batch_cnt,
- float *dist2, int *idx) {
- // unknown: (N1 + N2 ..., 3)
- // unknown_batch_cnt: (batch_size), [N1, N2, ...]
- // known: (M1 + M2 ..., 3)
- // known_batch_cnt: (batch_size), [M1, M2, ...]
- // Return:
- // dist: (N1 + N2 ..., 3) l2 distance to the three nearest neighbors
- // idx: (N1 + N2 ..., 3) index of the three nearest neighbors
-
- int pt_idx = blockIdx.x * blockDim.x + threadIdx.x;
- if (pt_idx >= N) return;
-
- int bs_idx = 0, pt_cnt = unknown_batch_cnt[0];
- for (int k = 1; k < batch_size; k++){
- if (pt_idx < pt_cnt) break;
- pt_cnt += unknown_batch_cnt[k];
- bs_idx = k;
- }
-
- int cur_num_known_points = known_batch_cnt[bs_idx];
-
- int known_batch_start_idx = 0;
- for (int k = 0; k < bs_idx; k++) known_batch_start_idx += known_batch_cnt[k];
-
- known += known_batch_start_idx * 3;
- unknown += pt_idx * 3;
- dist2 += pt_idx * 3;
- idx += pt_idx * 3;
-
- float ux = unknown[0];
- float uy = unknown[1];
- float uz = unknown[2];
-
- double best1 = 1e40, best2 = 1e40, best3 = 1e40;
- int besti1 = 0, besti2 = 0, besti3 = 0;
- for (int k = 0; k < cur_num_known_points; ++k) {
- float x = known[k * 3 + 0];
- float y = known[k * 3 + 1];
- float z = known[k * 3 + 2];
- float d = (ux - x) * (ux - x) + (uy - y) * (uy - y) + (uz - z) * (uz - z);
- if (d < best1) {
- best3 = best2; besti3 = besti2;
- best2 = best1; besti2 = besti1;
- best1 = d; besti1 = k;
- }
- else if (d < best2) {
- best3 = best2; besti3 = besti2;
- best2 = d; besti2 = k;
- }
- else if (d < best3) {
- best3 = d; besti3 = k;
- }
- }
- dist2[0] = best1; dist2[1] = best2; dist2[2] = best3;
- idx[0] = besti1 + known_batch_start_idx;
- idx[1] = besti2 + known_batch_start_idx;
- idx[2] = besti3 + known_batch_start_idx;
-}
-
-
-void three_nn_kernel_launcher_stack(int batch_size, int N, int M, const float *unknown,
- const int *unknown_batch_cnt, const float *known, const int *known_batch_cnt,
- float *dist2, int *idx) {
- // unknown: (N1 + N2 ..., 3)
- // unknown_batch_cnt: (batch_size), [N1, N2, ...]
- // known: (M1 + M2 ..., 3)
- // known_batch_cnt: (batch_size), [M1, M2, ...]
- // Return:
- // dist: (N1 + N2 ..., 3) l2 distance to the three nearest neighbors
- // idx: (N1 + N2 ..., 3) index of the three nearest neighbors
-
- cudaError_t err;
- dim3 blocks(DIVUP(N, THREADS_PER_BLOCK)); // blockIdx.x(col), blockIdx.y(row)
- dim3 threads(THREADS_PER_BLOCK);
-
- three_nn_kernel_stack<<>>(
- batch_size, N, M, unknown, unknown_batch_cnt,
- known, known_batch_cnt, dist2, idx
- );
-
- err = cudaGetLastError();
- if (cudaSuccess != err) {
- fprintf(stderr, "CUDA kernel failed : %s\n", cudaGetErrorString(err));
- exit(-1);
- }
-}
-
-
-
-__global__ void three_interpolate_kernel_stack(int N, int channels, const float *features,
- const int *idx, const float *weight, float *out) {
- // features: (M1 + M2 ..., C)
- // idx: [N1 + N2 ..., 3]
- // weight: [N1 + N2 ..., 3]
- // Return:
- // out: (N1 + N2 ..., C)
-
- int c_idx = blockIdx.y;
- int pt_idx = blockIdx.x * blockDim.x + threadIdx.x;
- if (pt_idx >= N || c_idx >= channels) return;
-
- weight += pt_idx * 3;
- idx += pt_idx * 3;
- out += pt_idx * channels + c_idx;
-
- out[0] = weight[0] * features[idx[0] * channels + c_idx] +
- weight[1] * features[idx[1] * channels + c_idx] +
- weight[2] * features[idx[2] * channels + c_idx];
-}
-
-
-
-void three_interpolate_kernel_launcher_stack(int N, int channels,
- const float *features, const int *idx, const float *weight, float *out) {
- // features: (M1 + M2 ..., C)
- // idx: [N1 + N2 ..., 3]
- // weight: [N1 + N2 ..., 3]
- // Return:
- // out: (N1 + N2 ..., C)
-
- cudaError_t err;
- dim3 blocks(DIVUP(N, THREADS_PER_BLOCK), channels);
- dim3 threads(THREADS_PER_BLOCK);
- three_interpolate_kernel_stack<<>>(N, channels, features, idx, weight, out);
-
- err = cudaGetLastError();
- if (cudaSuccess != err) {
- fprintf(stderr, "CUDA kernel failed : %s\n", cudaGetErrorString(err));
- exit(-1);
- }
-}
-
-
-__global__ void three_interpolate_grad_kernel_stack(int N, int channels, const float *grad_out,
- const int *idx, const float *weight, float *grad_features) {
- // grad_out_tensor: (N1 + N2 ..., C)
- // idx_tensor: [N1 + N2 ..., 3]
- // weight_tensor: [N1 + N2 ..., 3]
- // Return:
- // grad_features_tensor: (M1 + M2 ..., C)
-
- int c_idx = blockIdx.y;
- int pt_idx = blockIdx.x * blockDim.x + threadIdx.x;
- if (pt_idx >= N || c_idx >= channels) return;
-
- grad_out += pt_idx * channels + c_idx;
- weight += pt_idx * 3;
- idx += pt_idx * 3;
-
- // printf("pt_idx=%d, c_idx=%d, idx=(%d, %d, %d), grad_out=%f\n", pt_idx, c_idx, idx[0], idx[1], idx[2], grad_out[0]);
-
- atomicAdd(grad_features + idx[0] * channels + c_idx, grad_out[0] * weight[0]);
- atomicAdd(grad_features + idx[1] * channels + c_idx, grad_out[0] * weight[1]);
- atomicAdd(grad_features + idx[2] * channels + c_idx, grad_out[0] * weight[2]);
-}
-
-
-void three_interpolate_grad_kernel_launcher_stack(int N, int channels, const float *grad_out,
- const int *idx, const float *weight, float *grad_features) {
- // grad_out_tensor: (N1 + N2 ..., C)
- // idx_tensor: [N1 + N2 ..., 3]
- // weight_tensor: [N1 + N2 ..., 3]
- // Return:
- // grad_features_tensor: (M1 + M2 ..., C)
-
- cudaError_t err;
- dim3 blocks(DIVUP(N, THREADS_PER_BLOCK), channels); // blockIdx.x(col), blockIdx.y(row)
- dim3 threads(THREADS_PER_BLOCK);
- three_interpolate_grad_kernel_stack<<>>(
- N, channels, grad_out, idx, weight, grad_features
- );
-
- err = cudaGetLastError();
- if (cudaSuccess != err) {
- fprintf(stderr, "CUDA kernel failed : %s\n", cudaGetErrorString(err));
- exit(-1);
- }
-}
diff --git a/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_stack/src/interpolate_gpu.h b/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_stack/src/interpolate_gpu.h
deleted file mode 100644
index 016c7886..00000000
--- a/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_stack/src/interpolate_gpu.h
+++ /dev/null
@@ -1,39 +0,0 @@
-#ifndef _INTERPOLATE_GPU_H
-#define _INTERPOLATE_GPU_H
-
-#include
-#include
-#include
-#include
-
-
-void three_nn_wrapper_stack(at::Tensor unknown_tensor,
- at::Tensor unknown_batch_cnt_tensor, at::Tensor known_tensor,
- at::Tensor known_batch_cnt_tensor, at::Tensor dist2_tensor, at::Tensor idx_tensor);
-
-
-void three_interpolate_wrapper_stack(at::Tensor features_tensor,
- at::Tensor idx_tensor, at::Tensor weight_tensor, at::Tensor out_tensor);
-
-
-
-void three_interpolate_grad_wrapper_stack(at::Tensor grad_out_tensor, at::Tensor idx_tensor,
- at::Tensor weight_tensor, at::Tensor grad_features_tensor);
-
-
-void three_nn_kernel_launcher_stack(int batch_size, int N, int M, const float *unknown,
- const int *unknown_batch_cnt, const float *known, const int *known_batch_cnt,
- float *dist2, int *idx);
-
-
-void three_interpolate_kernel_launcher_stack(int N, int channels,
- const float *features, const int *idx, const float *weight, float *out);
-
-
-
-void three_interpolate_grad_kernel_launcher_stack(int N, int channels, const float *grad_out,
- const int *idx, const float *weight, float *grad_features);
-
-
-
-#endif
diff --git a/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_stack/src/pointnet2_api.cpp b/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_stack/src/pointnet2_api.cpp
deleted file mode 100644
index 591f11a7..00000000
--- a/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_stack/src/pointnet2_api.cpp
+++ /dev/null
@@ -1,21 +0,0 @@
-#include
-#include
-
-#include "ball_query_gpu.h"
-#include "group_points_gpu.h"
-#include "sampling_gpu.h"
-#include "interpolate_gpu.h"
-
-
-PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
- m.def("ball_query_wrapper", &ball_query_wrapper_stack, "ball_query_wrapper_stack");
-
- m.def("furthest_point_sampling_wrapper", &furthest_point_sampling_wrapper, "furthest_point_sampling_wrapper");
-
- m.def("group_points_wrapper", &group_points_wrapper_stack, "group_points_wrapper_stack");
- m.def("group_points_grad_wrapper", &group_points_grad_wrapper_stack, "group_points_grad_wrapper_stack");
-
- m.def("three_nn_wrapper", &three_nn_wrapper_stack, "three_nn_wrapper_stack");
- m.def("three_interpolate_wrapper", &three_interpolate_wrapper_stack, "three_interpolate_wrapper_stack");
- m.def("three_interpolate_grad_wrapper", &three_interpolate_grad_wrapper_stack, "three_interpolate_grad_wrapper_stack");
-}
diff --git a/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_stack/src/sampling.cpp b/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_stack/src/sampling.cpp
deleted file mode 100644
index 6987cae0..00000000
--- a/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_stack/src/sampling.cpp
+++ /dev/null
@@ -1,24 +0,0 @@
-#include
-#include
-#include
-#include "sampling_gpu.h"
-
-
-#define CHECK_CUDA(x) TORCH_CHECK(x.is_cuda(), #x " must be a CUDA tensor")
-#define CHECK_CONTIGUOUS(x) TORCH_CHECK(x.is_contiguous(), #x " must be contiguous")
-#define CHECK_INPUT(x) CHECK_CUDA(x); CHECK_CONTIGUOUS(x)
-
-int furthest_point_sampling_wrapper(int b, int n, int m,
- at::Tensor points_tensor, at::Tensor temp_tensor, at::Tensor idx_tensor) {
-
- CHECK_INPUT(points_tensor);
- CHECK_INPUT(temp_tensor);
- CHECK_INPUT(idx_tensor);
-
- const float* points = points_tensor.data_ptr();
- float* temp = temp_tensor.data_ptr();
- int* idx = idx_tensor.data_ptr();
-
- furthest_point_sampling_kernel_launcher(b, n, m, points, temp, idx);
- return 1;
-}
diff --git a/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_stack/src/sampling_gpu.cu b/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_stack/src/sampling_gpu.cu
deleted file mode 100644
index 02e8f3bf..00000000
--- a/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_stack/src/sampling_gpu.cu
+++ /dev/null
@@ -1,184 +0,0 @@
-#include
-#include
-
-#include "cuda_utils.h"
-#include "sampling_gpu.h"
-#define TOTAL_THREADS 1024
-
-
-inline int opt_n_threads(int work_size) {
- const int pow_2 = std::log(static_cast(work_size)) / std::log(2.0);
-
- return max(min(1 << pow_2, TOTAL_THREADS), 1);
-}
-
-
-__device__ void __update(float *__restrict__ dists, int *__restrict__ dists_i, int idx1, int idx2){
- const float v1 = dists[idx1], v2 = dists[idx2];
- const int i1 = dists_i[idx1], i2 = dists_i[idx2];
- dists[idx1] = max(v1, v2);
- dists_i[idx1] = v2 > v1 ? i2 : i1;
-}
-
-
-template
-__global__ void furthest_point_sampling_kernel(int b, int n, int m,
- const float *__restrict__ dataset, float *__restrict__ temp, int *__restrict__ idxs) {
- // dataset: (B, N, 3)
- // tmp: (B, N)
- // output:
- // idx: (B, M)
-
- if (m <= 0) return;
- __shared__ float dists[block_size];
- __shared__ int dists_i[block_size];
-
- int batch_index = blockIdx.x;
- dataset += batch_index * n * 3;
- temp += batch_index * n;
- idxs += batch_index * m;
-
- int tid = threadIdx.x;
- const int stride = block_size;
-
- int old = 0;
- if (threadIdx.x == 0)
- idxs[0] = old;
-
- __syncthreads();
- for (int j = 1; j < m; j++) {
- int besti = 0;
- float best = -1;
- float x1 = dataset[old * 3 + 0];
- float y1 = dataset[old * 3 + 1];
- float z1 = dataset[old * 3 + 2];
- for (int k = tid; k < n; k += stride) {
- float x2, y2, z2;
- x2 = dataset[k * 3 + 0];
- y2 = dataset[k * 3 + 1];
- z2 = dataset[k * 3 + 2];
- // float mag = (x2 * x2) + (y2 * y2) + (z2 * z2);
- // if (mag <= 1e-3)
- // continue;
-
- float d = (x2 - x1) * (x2 - x1) + (y2 - y1) * (y2 - y1) + (z2 - z1) * (z2 - z1);
- float d2 = min(d, temp[k]);
- temp[k] = d2;
- besti = d2 > best ? k : besti;
- best = d2 > best ? d2 : best;
- }
- dists[tid] = best;
- dists_i[tid] = besti;
- __syncthreads();
-
- if (block_size >= 1024) {
- if (tid < 512) {
- __update(dists, dists_i, tid, tid + 512);
- }
- __syncthreads();
- }
-
- if (block_size >= 512) {
- if (tid < 256) {
- __update(dists, dists_i, tid, tid + 256);
- }
- __syncthreads();
- }
- if (block_size >= 256) {
- if (tid < 128) {
- __update(dists, dists_i, tid, tid + 128);
- }
- __syncthreads();
- }
- if (block_size >= 128) {
- if (tid < 64) {
- __update(dists, dists_i, tid, tid + 64);
- }
- __syncthreads();
- }
- if (block_size >= 64) {
- if (tid < 32) {
- __update(dists, dists_i, tid, tid + 32);
- }
- __syncthreads();
- }
- if (block_size >= 32) {
- if (tid < 16) {
- __update(dists, dists_i, tid, tid + 16);
- }
- __syncthreads();
- }
- if (block_size >= 16) {
- if (tid < 8) {
- __update(dists, dists_i, tid, tid + 8);
- }
- __syncthreads();
- }
- if (block_size >= 8) {
- if (tid < 4) {
- __update(dists, dists_i, tid, tid + 4);
- }
- __syncthreads();
- }
- if (block_size >= 4) {
- if (tid < 2) {
- __update(dists, dists_i, tid, tid + 2);
- }
- __syncthreads();
- }
- if (block_size >= 2) {
- if (tid < 1) {
- __update(dists, dists_i, tid, tid + 1);
- }
- __syncthreads();
- }
-
- old = dists_i[0];
- if (tid == 0)
- idxs[j] = old;
- }
-}
-
-void furthest_point_sampling_kernel_launcher(int b, int n, int m,
- const float *dataset, float *temp, int *idxs) {
- // dataset: (B, N, 3)
- // tmp: (B, N)
- // output:
- // idx: (B, M)
-
- cudaError_t err;
- unsigned int n_threads = opt_n_threads(n);
-
- switch (n_threads) {
- case 1024:
- furthest_point_sampling_kernel<1024><<>>(b, n, m, dataset, temp, idxs); break;
- case 512:
- furthest_point_sampling_kernel<512><<>>(b, n, m, dataset, temp, idxs); break;
- case 256:
- furthest_point_sampling_kernel<256><<>>(b, n, m, dataset, temp, idxs); break;
- case 128:
- furthest_point_sampling_kernel<128><<>>(b, n, m, dataset, temp, idxs); break;
- case 64:
- furthest_point_sampling_kernel<64><<>>(b, n, m, dataset, temp, idxs); break;
- case 32:
- furthest_point_sampling_kernel<32><<>>(b, n, m, dataset, temp, idxs); break;
- case 16:
- furthest_point_sampling_kernel<16><<>>(b, n, m, dataset, temp, idxs); break;
- case 8:
- furthest_point_sampling_kernel<8><<>>(b, n, m, dataset, temp, idxs); break;
- case 4:
- furthest_point_sampling_kernel<4><<>>(b, n, m, dataset, temp, idxs); break;
- case 2:
- furthest_point_sampling_kernel<2><<>>(b, n, m, dataset, temp, idxs); break;
- case 1:
- furthest_point_sampling_kernel<1><<>>(b, n, m, dataset, temp, idxs); break;
- default:
- furthest_point_sampling_kernel<512><<>>(b, n, m, dataset, temp, idxs);
- }
-
- err = cudaGetLastError();
- if (cudaSuccess != err) {
- fprintf(stderr, "CUDA kernel failed : %s\n", cudaGetErrorString(err));
- exit(-1);
- }
-}
diff --git a/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_stack/src/sampling_gpu.h b/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_stack/src/sampling_gpu.h
deleted file mode 100644
index 2ad8aa5b..00000000
--- a/OpenCOOD/opencood/pcdet_utils/pointnet2/pointnet2_stack/src/sampling_gpu.h
+++ /dev/null
@@ -1,15 +0,0 @@
-#ifndef _SAMPLING_GPU_H
-#define _SAMPLING_GPU_H
-
-#include
-#include
-#include
-
-
-int furthest_point_sampling_wrapper(int b, int n, int m,
- at::Tensor points_tensor, at::Tensor temp_tensor, at::Tensor idx_tensor);
-
-void furthest_point_sampling_kernel_launcher(int b, int n, int m,
- const float *dataset, float *temp, int *idxs);
-
-#endif
diff --git a/OpenCOOD/opencood/pcdet_utils/roiaware_pool3d/roiaware_pool3d_utils.py b/OpenCOOD/opencood/pcdet_utils/roiaware_pool3d/roiaware_pool3d_utils.py
deleted file mode 100644
index 6383cc34..00000000
--- a/OpenCOOD/opencood/pcdet_utils/roiaware_pool3d/roiaware_pool3d_utils.py
+++ /dev/null
@@ -1,55 +0,0 @@
-import torch
-
-from opencood.utils import common_utils
-from opencood.pcdet_utils.roiaware_pool3d import roiaware_pool3d_cuda
-
-
-def points_in_boxes_cpu(points, boxes):
- """
- Args:
- points: (num_points, 3)
- boxes: [x, y, z, dx, dy, dz, heading], (x, y, z) is the box center, each box DO NOT overlaps
- Returns:
- point_indices: (N, num_points)
- """
- assert boxes.shape[1] == 7
- assert points.shape[1] == 3
- points, is_numpy = common_utils.check_numpy_to_torch(points)
- boxes, is_numpy = common_utils.check_numpy_to_torch(boxes)
-
- point_indices = points.new_zeros((boxes.shape[0], points.shape[0]), dtype=torch.int)
- roiaware_pool3d_cuda.points_in_boxes_cpu(boxes.float().contiguous(), points.float().contiguous(), point_indices)
-
- return point_indices.numpy() if is_numpy else point_indices
-
-
-def points_in_boxes_gpu(points, boxes):
- """
- :param points: (B, M, 3)
- :param boxes: (B, T, 7), num_valid_boxes <= T
- :return box_idxs_of_pts: (B, M), default background = -1
- """
- assert boxes.shape[0] == points.shape[0]
- assert boxes.shape[2] == 7 and points.shape[2] == 3
- # #######
- # import matplotlib.pyplot as plt
- # ax = plt.figure(figsize=(8, 8)).add_subplot(1, 1, 1)
- # ax.set_aspect('equal', 'box')
- # ax.set(xlim=(-50, 50),
- # ylim=(-41.6, 41.6))
- # points0 = points[0].cpu().detach().numpy()
- # boxes0 = boxes[0].cpu().detach().numpy()
- # ax.plot(points0[:, 0], points0[:, 1], 'y.', markersize=3)
- # ax.plot(boxes0[:, 0], boxes0[:, 1], 'r.', markersize=10)
- # plt.xlabel('x')
- # plt.ylabel('y')
- #
- # plt.show()
- # plt.close()
- # ########
- batch_size, num_points, _ = points.shape
-
- box_idxs_of_pts = points.new_zeros((batch_size, num_points), dtype=torch.int).fill_(-1)
- roiaware_pool3d_cuda.points_in_boxes_gpu(boxes.contiguous(), points.contiguous(), box_idxs_of_pts)
-
- return box_idxs_of_pts
diff --git a/OpenCOOD/opencood/pcdet_utils/roiaware_pool3d/src/roiaware_pool3d.cpp b/OpenCOOD/opencood/pcdet_utils/roiaware_pool3d/src/roiaware_pool3d.cpp
deleted file mode 100644
index e191c491..00000000
--- a/OpenCOOD/opencood/pcdet_utils/roiaware_pool3d/src/roiaware_pool3d.cpp
+++ /dev/null
@@ -1,177 +0,0 @@
-/*
-RoI-aware point cloud feature pooling
-Reference paper: https://arxiv.org/abs/1907.03670
-Written by Shaoshuai Shi
-All Rights Reserved 2019-2020.
-*/
-
-
-#include
-#include
-#include
-
-
-//#define CHECK_CUDA(x) AT_CHECK(x.type().is_cuda(), #x, " must be a CUDAtensor ")
-//#define CHECK_CONTIGUOUS(x) AT_CHECK(x.is_contiguous(), #x, " must be contiguous ")
-//#define CHECK_INPUT(x) CHECK_CUDA(x);CHECK_CONTIGUOUS(x)
-
-
-void roiaware_pool3d_launcher(int boxes_num, int pts_num, int channels, int max_pts_each_voxel,
- int out_x, int out_y, int out_z, const float *rois, const float *pts, const float *pts_feature,
- int *argmax, int *pts_idx_of_voxels, float *pooled_features, int pool_method);
-
-void roiaware_pool3d_backward_launcher(int boxes_num, int out_x, int out_y, int out_z, int channels, int max_pts_each_voxel,
- const int *pts_idx_of_voxels, const int *argmax, const float *grad_out, float *grad_in, int pool_method);
-
-void points_in_boxes_launcher(int batch_size, int boxes_num, int pts_num, const float *boxes,
- const float *pts, int *box_idx_of_points);
-
-int roiaware_pool3d_gpu(at::Tensor rois, at::Tensor pts, at::Tensor pts_feature, at::Tensor argmax,
- at::Tensor pts_idx_of_voxels, at::Tensor pooled_features, int pool_method){
- // params rois: (N, 7) [x, y, z, dx, dy, dz, heading] (x, y, z) is the box center
- // params pts: (npoints, 3) [x, y, z]
- // params pts_feature: (npoints, C)
- // params argmax: (N, out_x, out_y, out_z, C)
- // params pts_idx_of_voxels: (N, out_x, out_y, out_z, max_pts_each_voxel)
- // params pooled_features: (N, out_x, out_y, out_z, C)
- // params pool_method: 0: max_pool 1: avg_pool
-
-// CHECK_INPUT(rois);
-// CHECK_INPUT(pts);
-// CHECK_INPUT(pts_feature);
-// CHECK_INPUT(argmax);
-// CHECK_INPUT(pts_idx_of_voxels);
-// CHECK_INPUT(pooled_features);
-
- int boxes_num = rois.size(0);
- int pts_num = pts.size(0);
- int channels = pts_feature.size(1);
- int max_pts_each_voxel = pts_idx_of_voxels.size(4); // index 0 is the counter
- int out_x = pts_idx_of_voxels.size(1);
- int out_y = pts_idx_of_voxels.size(2);
- int out_z = pts_idx_of_voxels.size(3);
- assert ((out_x < 256) && (out_y < 256) && (out_z < 256)); // we encode index with 8bit
-
- const float *rois_data = rois.data_ptr();
- const float *pts_data = pts.data_ptr