Turned cangen into a module, and refactored cangen (#233)

* packaged can_generator into python package

* Added object for config and busses, fixed cangen

* refactored module and restructured some code

* used pop to parse config.yaml

* updated readme to be more clear

* changed name of config

* added pop to consume config.yaml

* made readme more clear

* fixed can_generator errors

* Update cmake to use CANgen and change install docs

* Adds note about editable install

* Reincorporates Clean Cangen

---------

Co-authored-by: BlakeFreer <[email protected]>

docs/docs/firmware/dev-setup.md

@@ -150,7 +150,7 @@ Navigate to a directory where you would like to hold the `racecar` repo (I used
 
     git clone --recurse-submodules https://github.com/macformula/racecar.git
 
-## Install CANgen dependencies
+## Install CANgen
 
 Create a Python virtual environment for CANgen. Navigate into `racecar/firmware` and run.
 
@@ -168,14 +168,16 @@ Create a Python virtual environment for CANgen. Navigate into `racecar/firmware`
     source .env/bin/activate
     ```
 
-The second command "activates" the virtual environment. You will see `(.env)` beside your terminal prompt when it is activated, and it must be activated before building any project.
+The second command "activates" the virtual environment. You will see `(.env)` beside your terminal prompt when it is activated. __It must be activated before building any project.__
 
-Finally, with the environment activated, install the CANgen requirements.
+With the environment activated, change into `racecar/` and install CANgen:
 
 ```bash
-pip install -r ../scripts/cangen/requirements.txt
+pip install -e scripts/cangen
 ```
 
+> The `-e` flag is _very_ important. It install CANgen as an editable package which means you won't have to reinstall when the package is changed.
+
 You can now start developing in `racecar`! However, I recommend you configure your IDE with `clangd`, so continue to the next section.
 
 ## IDE Integration

firmware/cmake/cangen.cmake

@@ -1,22 +1,14 @@
-find_package(Python3 COMPONENTS Interpreter)
-message(${Python3_EXECUTABLE})
-
-if(NOT ${Python3_FOUND})
-	message(FATAL_ERROR "Python 3 executable not found")
-endif()
-
-FILE(GLOB_RECURSE CAN_DEPENDENCIES CONFIGURE_DEPENDS "${CMAKE_SOURCE_DIR}/../scripts/cangen/*" "${CMAKE_SOURCE_DIR}/dbcs/*")
-
 list(APPEND CAN_DEPENDENCIES "${CMAKE_CURRENT_SOURCE_DIR}/config.yaml")
 
 cmake_language(GET_MESSAGE_LOG_LEVEL LOG_LEVEL)
 
+# cangen is provided by racecar/scripts/cangen
 add_custom_target(
     generated_can
-	COMMAND ${Python3_EXECUTABLE} "${CMAKE_SOURCE_DIR}/../scripts/cangen/main.py" "--project=\"${CMAKE_CURRENT_SOURCE_DIR}\"" "--log-level=${LOG_LEVEL}"
-	DEPENDS ${CAN_DEPENDENCIES}
-	WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-	COMMENT Generating CAN code from DBCs
+    COMMAND "cangen" ${CMAKE_CURRENT_SOURCE_DIR} "--log-level=${LOG_LEVEL}"
+    DEPENDS ${CAN_DEPENDENCIES}
+    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+    COMMENT Generating CAN code from DBCs
 )
 
 add_dependencies(main generated_can)

firmware/projects/EV5/FrontController/config.yaml

@@ -2,8 +2,6 @@ canGen:
   ourNode: fc
   busses:
     - busName: veh
-      dbcFiles:
-        - "../veh.dbc"
+      dbcFile: "../veh.dbc"
     - busName: pt
-      dbcFiles:
-        - "../pt.dbc"
+      dbcFile: "../pt.dbc"

firmware/projects/EV5/LVController/config.yaml

@@ -2,5 +2,4 @@ canGen:
   ourNode: lvc
   busses:
     - busName: veh
-      dbcFiles:
-        - "../veh.dbc"
+      dbcFile: "../veh.dbc"

firmware/projects/EV5/TMS/config.yaml

@@ -2,5 +2,4 @@ canGen:
   ourNode: tms
   busses:
     - busName: veh
-      dbcFiles:
-        - "../veh.dbc"
+      dbcFile: "../veh.dbc"

firmware/projects/EV5/debug/FrontControllerSimple/config.yaml

@@ -2,8 +2,6 @@ canGen:
   ourNode: fc
   busses:
     - busName: veh
-      dbcFiles:
-        - "../../veh.dbc"
+      dbcFile: "../../veh.dbc"
     - busName: pt
-      dbcFiles:
-        - "../../pt.dbc"
+      dbcFile: "../../pt.dbc"

firmware/projects/EV5/debug/IoCheckoutFc/config.yaml

@@ -2,5 +2,4 @@ canGen:
   ourNode: FC
   busses:
     - busName: io
-      dbcFiles:
-        - "io.dbc"
+      dbcFile: "io.dbc"

firmware/projects/EV5/debug/MotorDebug/config.yaml

@@ -2,5 +2,4 @@ canGen:
   ourNode: FRONTCONTROLLER
   busses:
     - busName: vehicle
-      dbcFiles:
-        - "pedal.dbc"
+      dbcFile: "pedal.dbc"

scripts/cangen/README.md

@@ -0,0 +1,43 @@
+# Cangen
+
+Generates C++ files from a corresponding dbc file. Ensures repeatable generation between all projects.
+
+## Installation
+
+From the `racecar/` directory, run
+
+```bash
+pip install scripts/cangen
+```
+
+## Usage
+
+To generate CAN code for a project, execute `cangen` and pass the project folder. The project folder is the one which contains `config.yaml`.
+
+## Example
+
+If you are in the `racecar/firmware/` directory, you could generate `EV5/FrontController` code with
+
+```bash
+cangen projects/EV5/FrontController
+```
+
+This command will generate code in a `generated/can/` subfolder of the project.
+
+```
+FrontController
+├─ fc_docs
+├─ generated/can
+│  ├─ .gitignore
+│  ├─ pt_can_messages.h
+│  ├─ pt_msg_registry.h
+│  ├─ veh_can_messages.h
+│  └─ veh_msg_registry.h
+├─ inc
+├─ platforms
+├─ vehicle_control_system
+├─ CMakeLists.txt
+├─ config.yaml
+├─ main.cc
+└─ README.md
+```

scripts/cangen/can_generator.py → scripts/cangen/cangen/can_generator.py

@@ -7,13 +7,16 @@
 import math
 import os
 import re
+import shutil
 import time
 from typing import Dict, List, Tuple
 
 import numpy as np
 from cantools.database import Database, Message, Signal
 from jinja2 import Environment
 
+from .config import Bus, Config
+
 logger = logging.getLogger(__name__)
 
 EIGHT_BITS = 8
@@ -22,43 +25,45 @@
 MSG_REGISTRY_FILE_NAME = "_msg_registry.h"
 CAN_MESSAGES_FILE_NAME = "_can_messages.h"
 
+CAN_MESSAGES_TEMPLATE_FILENAME = "can_messages.h.jinja2"
+MSG_REGISTRY_TEMPLATE_FILENAME = "msg_registry.h.jinja2"
 
-def _assert_valid_dbc(filename: str):
-    """Raise an error if filename is not a valid and existant dbc file."""
-
-    if not os.path.isfile(filename):
-        raise FileNotFoundError(f"Could not find a file at {filename}.")
-
-    _, extension = os.path.splitext(filename)
-    if extension != ".dbc":
-        raise ValueError(f"{filename} is not a .dbc file.")
+DIR_THIS_FILE = os.path.abspath(os.path.dirname(__file__))
+DIR_TEMPLATES = os.path.join(DIR_THIS_FILE, "templates")
+CAN_MESSAGES_TEMPLATE_PATH = os.path.join(DIR_TEMPLATES, CAN_MESSAGES_TEMPLATE_FILENAME)
+MSG_REGISTRY_TEMPLATE_PATH = os.path.join(DIR_TEMPLATES, MSG_REGISTRY_TEMPLATE_FILENAME)
 
-    logger.debug(f"{filename} is a valid dbc file.")
 
-
-def _parse_dbc_files(dbc_files: List[str]) -> Database:
-    logger.info(f"Parsing DBC files: {dbc_files}")
+def _parse_dbc_files(dbc_file: str) -> Database:
+    logger.info(f"Parsing DBC files: {dbc_file}")
     can_db = Database()
 
-    for dbc_file in dbc_files:
-        _assert_valid_dbc(dbc_file)
-        with open(dbc_file, "r") as f:
-            can_db.add_dbc(f)
-            logger.info(f"Successfully added DBC: {dbc_file}")
+    with open(dbc_file, "r") as f:
+        can_db.add_dbc(f)
+        logger.info(f"Successfully added DBC: {dbc_file}")
 
     return can_db
 
-def _normalize_node_name(
-    node_name: str
-) -> str:
+
+def _normalize_node_name(node_name: str) -> str:
     return node_name.upper()
 
+
 def _filter_messages_by_node(
     messages: List[Message], node: str
 ) -> Tuple[List[Message], List[Message]]:
     normalized_node_name = _normalize_node_name(node)
-    tx_msgs = [msg for msg in messages if normalized_node_name in map(_normalize_node_name, msg.senders)]
-    rx_msgs = [msg for msg in messages if normalized_node_name in map(_normalize_node_name, msg.receivers)]
+
+    tx_msgs = [
+        msg
+        for msg in messages
+        if normalized_node_name in map(_normalize_node_name, msg.senders)
+    ]
+    rx_msgs = [
+        msg
+        for msg in messages
+        if normalized_node_name in map(_normalize_node_name, msg.receivers)
+    ]
 
     logger.debug(
         f"Filtered messages by node: {node}. "
@@ -91,7 +96,6 @@ def _get_mask_shift_big(
 def _get_mask_shift_little(
     length: int, start: int
 ) -> Tuple[np.ndarray[int], np.ndarray[int]]:
-
     idx = np.arange(64)
     mask_bool = (idx >= start) & (idx < start + length)
     mask_bytes = np.packbits(mask_bool, bitorder="little")
@@ -107,7 +111,6 @@ def _get_mask_shift_little(
 def _get_masks_shifts(
     msgs: List[Message],
 ) -> Dict[str, Dict[str, Tuple[List[int], List[int]]]]:
-
     # Create a dictionary of empty dictionaries, indexed by message names
     masks_shifts_dict = {msg.name: {} for msg in msgs}
 
@@ -151,7 +154,6 @@ def _get_signal_datatype(signal: Signal, allow_floating_point: bool = True) -> s
 
 
 def _get_signal_types(can_db: Database, allow_floating_point=True):
-
     # Create a dictionary (indexed by message name) of dictionaries (indexed by signal
     # name) corresponding to the datatype of each signal within each message.
     sig_types = {
@@ -174,20 +176,6 @@ def _camel_to_snake(text):
     return re.sub("([a-z0-9])([A-Z])", r"\1_\2", s1).lower()
 
 
-def _decimal_to_hex(decimal_value):
-    """
-    Converts a non-negative decimal integer to a lowercase hexadecimal string.
-
-    Raises:
-        ValueError: If the input is negative.
-    """
-
-    if decimal_value < 0:
-        raise ValueError("Input must be a non-negative integer")
-
-    return hex(decimal_value)
-
-
 def _generate_from_jinja2_template(
     template_path: str, output_path: str, context_dict: dict
 ):
@@ -200,7 +188,7 @@ def _generate_from_jinja2_template(
 
     # Register the camel_to_snake filter
     env.filters["camel_to_snake"] = _camel_to_snake
-    env.filters["decimal_to_hex"] = _decimal_to_hex
+    env.filters["decimal_to_hex"] = hex
 
     # Load the template from the string content
     template = env.from_string(template_str)
@@ -224,14 +212,7 @@ def _generate_from_jinja2_template(
     logger.info(f"Rendered code written to '{os.path.abspath(output_path)}'")
 
 
-def generate_code(
-    dbc_files: List[str],
-    our_node: str,
-    bus_name: str,
-    output_dir: str,
-    can_messages_template_path: str,
-    msg_registry_template_path: str,
-):
+def generate_code(bus: Bus, config: Config):
     """
     Parses DBC files, extracts information, and generates code using Jinja2
     templates.
@@ -241,9 +222,13 @@ def generate_code(
     templates (not included here) to create the final code.
     """
 
+    dbc_file = bus.dbc_file_path
+    our_node = config.node
+    bus_name = bus.bus_name
+
     logger.info("Generating code")
 
-    can_db = _parse_dbc_files(dbc_files)
+    can_db = _parse_dbc_files(dbc_file)
 
     signal_types = _get_signal_types(can_db)
     temp_signal_types = _get_signal_types(can_db, allow_floating_point=False)
@@ -266,19 +251,30 @@ def generate_code(
         "bus_name": bus_name,
     }
 
-    # Replace these lines with your Jinja2 template logic
     logger.debug("Generating code for can messages")
     _generate_from_jinja2_template(
-        can_messages_template_path,
-        os.path.join(output_dir, bus_name.lower() + CAN_MESSAGES_FILE_NAME),
+        CAN_MESSAGES_TEMPLATE_PATH,
+        os.path.join(config.output_dir, bus_name.lower() + CAN_MESSAGES_FILE_NAME),
         context,
     )
 
     logger.debug("Generating code for msg registry")
     _generate_from_jinja2_template(
-        msg_registry_template_path,
-        os.path.join(output_dir, bus_name.lower() + MSG_REGISTRY_FILE_NAME),
+        MSG_REGISTRY_TEMPLATE_PATH,
+        os.path.join(config.output_dir, bus_name.lower() + MSG_REGISTRY_FILE_NAME),
         context,
     )
 
     logger.info("Code generation complete")
+
+
+def generate_can_from_dbc(project_folder_name: str):
+    os.chdir(project_folder_name)
+    config = Config.from_yaml("config.yaml")
+
+    # Deletes output path folder and files within, before creating new ones
+    if os.path.exists(config.output_dir):
+        shutil.rmtree(config.output_dir)
+
+    for bus in config.busses:
+        generate_code(bus, config)

scripts/cangen/cangen/config.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+import yaml
+
+DEFAULT_OUTPUT_DIR = "generated/can"
+
+
+class Bus:
+    def __init__(self, bus: dict):
+        self.dbc_file_path: str = bus.pop("dbcFile")
+        self.bus_name: str = bus.pop("busName").capitalize()
+
+        if bus:
+            raise ValueError(
+                f"{bus.keys()} field/s not expected in configuration file for bus {self.bus_name}."
+            )
+
+
+class Config:
+    @staticmethod
+    def from_yaml(config_file_name: str) -> Config:
+        with open(config_file_name, "r") as file:
+            config = yaml.safe_load(file)
+        return Config(config.pop("canGen"))
+
+    def __init__(self, config: dict):
+        self.node = config.pop("ourNode")
+        self.output_dir = config.pop("outputPath", DEFAULT_OUTPUT_DIR)
+
+        self.busses = [Bus(bus) for bus in config.pop("busses")]
+
+        if config:
+            raise ValueError(
+                f"{config.keys()} field/s not expected in configuration file from node."
+            )