feat(CI): suppress generated files in PR diffs (#389)

* suppress generated code via .gitattributes

* add generate CI action to rebuild suppressed files

* add toolcheck

* bring in proto dependencies

* pin to golangci-lint 1.55 in generate workflow

* improve logs and remove toolcheck with dependencies installed

* test alteration of generated file

* remove generated grpc index.html docs

* remove more docs to test suppression

* remove comments from gencode to test suppression

* push generation after commit and improve logs

* add branch name to generate workflow

* add dummy proto to visualize suppressed diff

* .gitattributes update

* fix java generated code filepath

* denote the TODO to use groups instead of individuals in .gitattributes CODEOWNERship

* remove generate workflow in favor of using a github pages automation for docs and a schema registry for generated code during the hardening milestone work

* remove dummy proto

* make sure generation from protos cleans docs and regenerates

* clean codeowners

.gitattributes

@@ -0,0 +1,4 @@
+protocol/go/** linguist-generated=true
+sdkjava/src/main/java/** linguist-generated=true
+docs/grpc/** linguist-generated=true
+docs/openapi/** linguist-generated=true

.github/scripts/hsm-clean.sh

@@ -0,0 +1,16 @@
+#!/bin/sh
+
+# Deletes all you softhsm2 slots that have content. Use with caution
+
+set -ex
+
+PKCS11_MODULE_PATH=/lib/softhsm/libsofthsm2.so
+if which brew; then
+  PKCS11_MODULE_PATH=$(brew --prefix)/lib/softhsm/libsofthsm2.so
+fi
+
+softhsm2-util --show-slots | sed -n "s/^.*Serial number[^0-9a-f]*\([0-9a-f]*\)$/\1/p" | while read -r slot; do
+  if [ ! -z $slot ]; then
+    softhsm2-util --delete-token --serial $slot
+  fi
+done

.github/scripts/hsm-init-temporary-keys.sh

@@ -18,5 +18,6 @@ openssl ecparam -name prime256v1 >ecparams.tmp
 openssl req -x509 -nodes -newkey ec:ecparams.tmp -subj "/CN=kas" -keyout kas-ec-private.pem -out kas-ec-cert.pem -days 365
 pkcs11-tool --module $PKCS11_MODULE_PATH --login --pin 12345 --write-object kas-private.pem --type privkey --label development-rsa-kas
 pkcs11-tool --module $PKCS11_MODULE_PATH --login --pin 12345 --write-object kas-cert.pem --type cert --label development-rsa-kas
-pkcs11-tool --module $PKCS11_MODULE_PATH --login --pin 12345 --write-object kas-ec-private.pem --type privkey --label development-ec-kas
+# https://manpages.ubuntu.com/manpages/jammy/man1/pkcs11-tool.1.html --usage-derive
+pkcs11-tool --module $PKCS11_MODULE_PATH --login --pin 12345 --write-object kas-ec-private.pem --type privkey --label development-ec-kas --usage-derive
 pkcs11-tool --module $PKCS11_MODULE_PATH --login --pin 12345 --write-object kas-ec-cert.pem --type cert --label development-ec-kas

.github/workflows/checks.yaml

@@ -88,6 +88,16 @@ jobs:
       - run: go mod download
       - run: go mod verify
       - run: go test ./integration -race
+      - name: Install softHSM
+        run: |-
+          sudo apt-get install -y softhsm opensc openssl
+          sudo chmod +x /etc/softhsm
+          sudo chmod +r /etc/softhsm/softhsm2.conf
+          mkdir -p $(pwd)/.tmp/tokens
+          echo "directories.tokendir = $(pwd)/.tmp/tokens" > softhsm2.conf
+          echo "log.level = DEBUG" >> softhsm2.conf
+          echo "SOFTHSM2_CONF=$(pwd)/softhsm2.conf" >> "$GITHUB_ENV"
+      - run: .github/scripts/hsm-init-temporary-keys.sh
       - run: docker compose up -d --wait --wait-timeout 240
       - run: cp example-opentdf.yaml opentdf.yaml
       - uses: JarvusInnovations/background-action@313d37130873d82c33fc907b9b78e932aec8e990
@@ -102,6 +112,7 @@ jobs:
       - run: go install github.com/fullstorydev/grpcurl/cmd/[email protected]
       - run: grpcurl -plaintext localhost:9000 list
       - run: grpcurl -plaintext localhost:9000 grpc.health.v1.Health.Check
+      - run: curl --show-error --fail --insecure localhost:8080/kas/v2/kas_public_key
 
   image:
     name: image build
@@ -188,6 +199,7 @@ jobs:
         with:
           go-version: "1.21.8"
           cache: false
+      - run: make go.work
       - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36
         id: deps-changed
         with:
@@ -202,22 +214,22 @@ jobs:
         run: go install github.com/google/go-licenses@5348b744d0983d85713295ea08a20cca1654a45e
       - name: check platform licenses
         if: steps.deps-changed.outputs.platform == 'true'
-        run: >-
+        run: >
           go-licenses check --disallowed_types=forbidden --include_tests  
-              --ignore github.com/opentdf/platform/sdk
-              --ignore github.com/opentdf/platform/protocol
-              .
+          --ignore github.com/opentdf/platform/sdk
+          --ignore github.com/opentdf/platform/protocol
+          .
       - name: check sdk licenses
         if: steps.deps-changed.outputs.sdk == 'true'
-        run: >-
-          go-licenses check --disallowed_types=forbidden --include_tests  
-              --ignore github.com/opentdf/platform/sdk
-              --ignore github.com/opentdf/platform/protocol
-              ./sdk
+        run: >
+          go-licenses check --disallowed_types=forbidden --include_tests
+          --ignore github.com/opentdf/platform/sdk  
+          --ignore github.com/opentdf/platform/protocol
+          ./sdk
       - name: check examples licenses
         if: steps.deps-changed.outputs.examples == 'true'
-        run: >-
+        run: >
           go-licenses check --disallowed_types=forbidden --include_tests  
-              --ignore github.com/opentdf/platform/sdk
-              --ignore github.com/opentdf/platform/protocol
-              ./examples
+          --ignore github.com/opentdf/platform/sdk
+          --ignore github.com/opentdf/platform/protocol
+          ./examples

CODEOWNERS

@@ -1,3 +1,7 @@
 # CODEOWNERS
 
-*  @opentdf/developers
+* @opentdf/developers
+
+# Suppresses generated files in diffs
+# TODO: change these to known groups instead of individuals
+.gitattributes @biscoe916 @jrschumacher @strantalis @ttschampel

Makefile

@@ -11,6 +11,7 @@ EXCLUDE_JAVA=./services/authorization/idp_plugin.proto
 ROOT_DIR:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
 
 LINT_OPTIONS?=--new
+# LINT_OPTIONS?=-new-from-rev=main
 # LINT_OPTIONS?=-c $(ROOT_DIR)/.golangci-ratchet.yaml
 
 all: toolcheck clean build lint test
@@ -44,7 +45,7 @@ go-lint:
 	for m in $(MODS); do (cd $$m && golangci-lint run $(LINT_OPTIONS) --path-prefix=$$m) || exit 1; done
 
 proto-generate:
-	rm -rf sdkjava/src protocol/go/[a-fh-z]*
+	rm -rf sdkjava/src protocol/go/[a-fh-z]* docs/grpc docs/openapi
 	buf generate services
 	buf generate services --template buf.gen.grpc.docs.yaml
 	buf generate services --exclude-path $(EXCLUDE_JAVA) --template buf.gen.java.yaml