From d428c1f81d3252793c1c3c5fa047bc92f68079f0 Mon Sep 17 00:00:00 2001 From: Branden Cash <203336+ammmze@users.noreply.github.com> Date: Sat, 10 Sep 2022 17:39:50 -0700 Subject: [PATCH] feat: add gasket driver This driver enables the use of Coral.ai's PCIe TPU modules. Signed-off-by: Branden Cash <203336+ammmze@users.noreply.github.com> Signed-off-by: Noel Georgi --- Makefile | 14 ++++++++++- README.md | 13 +++++++--- drivers/gasket/README.md | 49 ++++++++++++++++++++++++++++++++++++ drivers/gasket/manifest.yaml | 12 +++++++++ drivers/gasket/pkg.yaml | 22 ++++++++++++++++ drivers/gasket/vars.yaml | 2 ++ hack/release.toml | 6 +++++ 7 files changed, 114 insertions(+), 4 deletions(-) create mode 100644 drivers/gasket/README.md create mode 100644 drivers/gasket/manifest.yaml create mode 100644 drivers/gasket/pkg.yaml create mode 100644 drivers/gasket/vars.yaml diff --git a/Makefile b/Makefile index 048d2b9e..0aadaab3 100644 --- a/Makefile +++ b/Makefile @@ -35,7 +35,19 @@ COMMON_ARGS += --build-arg=TAG=$(TAG) empty := space = $(empty) $(empty) -TARGETS = amd-ucode bnx2-bnx2x gvisor hello-world-service intel-ucode iscsi-tools nut-client nvidia-container-toolkit nvidia-fabricmanager nvidia-open-gpu-kernel-modules +TARGETS = \ + amd-ucode \ + bnx2-bnx2x \ + gasket-driver \ + gvisor \ + hello-world-service \ + intel-ucode \ + iscsi-tools \ + nut-client \ + nvidia-container-toolkit \ + nvidia-fabricmanager \ + nvidia-open-gpu-kernel-modules + NONFREE_TARGETS = all: $(TARGETS) ## Builds all known pkgs. diff --git a/README.md b/README.md index 195b550f..db46b617 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,13 @@ All system extensions provided by Sidero Labs can be found in the [ghcr.io regis | [bnx2-bnx2x](firmware/bnx2-bnx2x/) | [ghcr.io/siderolabs/bnx2-bnx2x](https://github.com/siderolabs/extensions/pkgs/container/bnx2-bnx2x) | Broadcom NetXtreme firmware | `linux firmware version` | | [intel-ucode](firmware/intel-ucode/) | [ghcr.io/siderolabs/intel-ucode](https://github.com/siderolabs/extensions/pkgs/container/intel-ucode) | Intel CPU microcode updates | `upstream version` | + +### Drivers +| Name | Image | Description | Version Format | +| ------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------ | ----------------------------------------------------- | +| [gasket](drivers/gasket/) | [ghcr.io/siderolabs/gasket-driver](https://github.com/siderolabs/extensions/pkgs/container/gasket-driver) | Driver for Google Coral PCIe devices | `gasket driver upstream short commit`-`talos version` | +| [nvidia](nvidia-gpu/nvidia-modules/) | [ghcr.io/siderolabs/nvidia-open-gpu-kernel-modules](https://github.com/siderolabs/extensions/pkgs/container/nvidia-open-gpu-kernel-modules) | NVIDIA OSS Driver | `nvidia driver upstream version`-`talos version` | + ### Storage | Name | Image | Description | Version Format | @@ -36,9 +43,9 @@ All system extensions provided by Sidero Labs can be found in the [ghcr.io regis ### Power -| Name | Image | Description | Version Format | -| ----------------------------------- | --------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------- | ---------------------------------- | -| [nut-client](power/nut-client/) | [ghcr.io/siderolabs/nut-client](https://github.com/siderolabs/talos-extensions/pkgs/container/nut-client) | [Network UPS Tools](https://networkupstools.org) upsmon client | `upstream version`-`talos version` | +| Name | Image | Description | Version Format | +| ------------------------------- | --------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------- | ---------------------------------- | +| [nut-client](power/nut-client/) | [ghcr.io/siderolabs/nut-client](https://github.com/siderolabs/talos-extensions/pkgs/container/nut-client) | [Network UPS Tools](https://networkupstools.org) upsmon client | `upstream version`-`talos version` | ### NVIDIA GPU diff --git a/drivers/gasket/README.md b/drivers/gasket/README.md new file mode 100644 index 00000000..f254edf0 --- /dev/null +++ b/drivers/gasket/README.md @@ -0,0 +1,49 @@ +# gasket-driver extension + +## Installation + +Add the extension to your machine config and enable the modules: + +```yaml +machine: + install: + extensions: + - image: ghcr.io/siderolabs/gasket-driver: + kernel: + modules: + - name: gasket + - name: apex +``` + +By default, the device will be owned by UID and GID `0` and is only accessible by root. +If you need to change this, you may do this by adding udev rules to your machine configuration like this, +which would change the GID to `44` and give that group read/write permissions. + +```yaml +machine: + udev: + rules: + - SUBSYSTEM=="apex", MODE="0660", GROUP="44" +``` + +## Verifiying + +You can verify the modules are enabled by reading the `/proc/modules` where it _should_ show the module is live. + +For example: + +``` +❯ talosctl -n 192.168.32.5 read /proc/modules +apex 20480 - - Live 0xffffffffc01c9000 (O) +gasket 94208 - - Live 0xffffffffc01aa000 (O) +``` + +In addition, if you actually have Coral module installed, you should be able to verify it's presence at `/dev/apex_0`. + +For example: + +``` +❯ talosctl -n 192.168.32.5 ls -l /dev/apex_0 +NODE MODE UID GID SIZE(B) LASTMOD NAME +192.168.32.5 Dcrw-rw---- 0 44 0 Sep 10 18:15:52 apex_0 +``` diff --git a/drivers/gasket/manifest.yaml b/drivers/gasket/manifest.yaml new file mode 100644 index 00000000..aa49d5b2 --- /dev/null +++ b/drivers/gasket/manifest.yaml @@ -0,0 +1,12 @@ +version: v1alpha1 +metadata: + name: gasket + version: "$VERSION" + author: Sidero Labs + description: | + This system extension provides google gasket driver kernel modules built against a specific Talos version. + This driver is required for PCIe and M.2 Google Coral accelerators. There are 2 kernel modules ("gasket" + and "apex") required to enable this driver. + compatibility: + talos: + version: ">= v1.2.0" diff --git a/drivers/gasket/pkg.yaml b/drivers/gasket/pkg.yaml new file mode 100644 index 00000000..bd736135 --- /dev/null +++ b/drivers/gasket/pkg.yaml @@ -0,0 +1,22 @@ +name: gasket-driver +variant: scratch +shell: /toolchain/bin/bash +dependencies: + - stage: base + # The pkgs version for a particular release of Talos as defined in + # https://github.com/siderolabs/talos/blob//pkg/machinery/gendata/data/pkgs + - image: "{{ .PKGS_PREFIX }}/gasket-driver-pkg:{{ .PKGS_VERSION }}" +steps: + - prepare: + - | + sed -i 's#$VERSION#{{ .VERSION }}#' /pkg/manifest.yaml + - install: + - | + mkdir -p /rootfs/lib/modules + + cp -R /lib/modules/* /rootfs/lib/modules +finalize: + - from: /rootfs + to: /rootfs + - from: /pkg/manifest.yaml + to: / diff --git a/drivers/gasket/vars.yaml b/drivers/gasket/vars.yaml new file mode 100644 index 00000000..16c27f18 --- /dev/null +++ b/drivers/gasket/vars.yaml @@ -0,0 +1,2 @@ +# update when pkgs version is updated +VERSION: "97aeba58-{{ .BUILD_ARG_TAG }}" diff --git a/hack/release.toml b/hack/release.toml index a177e2b4..d3ff7711 100644 --- a/hack/release.toml +++ b/hack/release.toml @@ -16,6 +16,12 @@ See [Talos Linux documentation](https://www.talos.dev/v1.2/talos-guides/configur [notes] + [notes.tpu] + title = "Coral.ai TPU gasket driver" + description = """\ +Extensions now publish the gasket driver for [Coral.ai](https://coral.ai/) PCIe TPU modules. +""" + [notes.updates] title = "Component Updates" description = """\