Merge pull request #101 from metal-stack/upload-to-gcp

Upload the firewall-controller binary to images.metal-stack.io.

.github/workflows/latest.yaml

@@ -24,7 +24,6 @@ jobs:
         sudo "PATH=$PATH" --preserve-env=GOROOT make test-all clean
 
   build:
-    needs: integration
     runs-on: ubuntu-20.04
     steps:
     - uses: actions/checkout@v2
@@ -42,14 +41,17 @@ jobs:
 
     - name: Build
       run: |
-        make
+        make fetch-template
 
     - name: Lint
       uses: golangci/golangci-lint-action@v2
       with:
-        version: v1.32.2
         args: --timeout=3m
 
+    - name: Build
+      run: |
+        make
+
     - name: Build and Push Docker Image
       run: |
         make docker-build

.github/workflows/pull_request.yaml

@@ -5,11 +5,15 @@ on:
     branches:
       - master
 
+env:
+  GCS_BUCKET: images.metal-pod.io
+
 jobs:
   integration:
     runs-on: ubuntu-20.04
     steps:
     - uses: actions/checkout@v2
+
     - name: Set up Go 1.16
       uses: actions/[email protected]
       with:
@@ -24,15 +28,23 @@ jobs:
         sudo "PATH=$PATH" --preserve-env make test-all clean
 
   build:
-    needs: integration
     runs-on: ubuntu-20.04
     steps:
     - uses: actions/checkout@v2
+
+    - uses: google-github-actions/setup-gcloud@master
+      with:
+        service_account_email: ${{ secrets.GCP_SA_EMAIL }}
+        service_account_key: ${{ secrets.GCP_SA_KEY }}
+
     - name: Set up Go 1.16
       uses: actions/[email protected]
       with:
         go-version: '1.16.x'
 
+    - name: Make tag
+      run: echo "TAG_NAME=$(echo $GITHUB_REF | awk -F / '{print $3}')-${GITHUB_HEAD_REF##*/}" >> $GITHUB_ENV
+
     - name: Figure out if running fork PR
       id: fork
       run: '["${{ secrets.DOCKER_REGISTRY_TOKEN }}" == ""] && echo "::set-output name=is_fork_pr::true" || echo "::set-output name=is_fork_pr::false"'
@@ -47,14 +59,17 @@ jobs:
 
     - name: Build
       run: |
-        make
+        make fetch-template
 
     - name: Lint
       uses: golangci/golangci-lint-action@v2
       with:
-        version: v1.32.2
         args: --timeout=3m
 
+    - name: Build
+      run: |
+        make
+
     - name: Build Docker Image
       run: |
         export GITHUB_TAG_NAME=${GITHUB_HEAD_REF##*/}
@@ -65,3 +80,14 @@ jobs:
         export GITHUB_TAG_NAME=${GITHUB_HEAD_REF##*/}
         make docker-push
       if: steps.fork.outputs.is_fork_pr == 'false'
+
+    - name: Prepare upload
+      run: |
+        mkdir -p firewall-controller/pull-requests/${TAG_NAME}
+        sudo chown -R $(id -u):$(id -g) bin
+        mv bin/* firewall-controller/pull-requests/${TAG_NAME}
+      if: steps.fork.outputs.is_fork_pr == 'false'
+
+    - name: Upload image tarballs to GCS
+      run: gsutil -m cp -r -p firewall-controller gs://$GCS_BUCKET
+      if: steps.fork.outputs.is_fork_pr == 'false'

.github/workflows/release.yaml

@@ -5,6 +5,9 @@ on:
     types:
       - published
 
+env:
+  GCS_BUCKET: images.metal-pod.io
+
 jobs:
   integration:
     runs-on: ubuntu-20.04
@@ -28,6 +31,12 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
     - uses: actions/checkout@v2
+
+    - uses: google-github-actions/setup-gcloud@master
+      with:
+        service_account_email: ${{ secrets.GCP_SA_EMAIL }}
+        service_account_key: ${{ secrets.GCP_SA_KEY }}
+
     - name: Set up Go 1.16
       uses: actions/[email protected]
       with:
@@ -48,14 +57,17 @@ jobs:
 
     - name: Build
       run: |
-        make
+        make fetch-template
 
     - name: Lint
       uses: golangci/golangci-lint-action@v2
       with:
-        version: v1.32.2
         args: --timeout=3m
 
+    - name: Build
+      run: |
+        make
+
     - name: Build and Push Docker Image
       run: |
         export GITHUB_TAG_NAME=${GITHUB_REF##*/}
@@ -82,3 +94,12 @@ jobs:
         asset_path: bin/firewall-controller.sha256
         asset_name: firewall-controller.sha256
         asset_content_type: application/octet-stream
+
+    - name: Prepare upload
+      run: |
+        mkdir -p firewall-controller/${GITHUB_TAG_NAME}/
+        sudo chown -R $(id -u):$(id -g) bin
+        mv bin/* firewall-controller/${GITHUB_TAG_NAME}/
+
+    - name: Upload image tarballs to GCS
+      run: gsutil -m cp -r -p firewall-controller gs://$GCS_BUCKET

api/v1/firewall_types.go

@@ -59,6 +59,8 @@ type FirewallSpec struct {
 	Signature string `json:"signature"`
 	// ControllerVersion holds the firewall-controller version to reconcile.
 	ControllerVersion string `json:"controllerVersion,omitempty"`
+	// ControllerURL points to the downloadable binary artifact of the firewall controller
+	ControllerURL string `json:"controllerURL,omitempty"`
 }
 
 // Data contains the fields over which the signature is calculated.

config/crd/bases/metal-stack.io_firewalls.yaml

@@ -45,6 +45,10 @@ spec:
           spec:
             description: FirewallSpec defines the desired state of Firewall
             properties:
+              controllerURL:
+                description: ControllerURL points to the downloadable binary artifact
+                  of the firewall controller
+                type: string
               controllerVersion:
                 description: ControllerVersion holds the firewall-controller version
                   to reconcile.

go.mod

@@ -7,8 +7,6 @@ require (
 	github.com/go-logr/logr v0.4.0
 	github.com/go-logr/zapr v0.3.0 // indirect
 	github.com/google/go-cmp v0.5.5
-	github.com/google/go-github v17.0.0+incompatible
-	github.com/google/go-querystring v1.0.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/nftables v0.0.0-20200802175506-c25e4f69b425
 	github.com/googleapis/gnostic v0.4.0 // indirect

go.sum

@@ -368,10 +368,6 @@ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
-github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
-github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
-github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=

pkg/updater/updater.go

@@ -1,29 +1,25 @@
 package updater
 
 import (
-	"context"
 	"crypto/sha256"
 	"encoding/hex"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"net/http"
+	"net/url"
 	"os"
 	"path/filepath"
 	"strings"
 
 	"github.com/go-logr/logr"
-	"github.com/google/go-github/github"
 	firewallv1 "github.com/metal-stack/firewall-controller/api/v1"
 	"github.com/metal-stack/v"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/client-go/tools/record"
 )
 
 const (
-	gitHubOwner    = "metal-stack"
-	gitHubRepo     = "firewall-controller"
-	gitHubArtifact = "firewall-controller"
 	binaryLocation = "/usr/local/bin/firewall-controller"
 )
 
@@ -38,15 +34,16 @@ func UpdateToSpecVersion(f firewallv1.Firewall, log logr.Logger, recorder record
 		return nil
 	}
 
-	recorder.Eventf(&f, corev1.EventTypeNormal, "Self-Reconcilation", "replacing firewall-controller version %s with version %s", v.Version, f.Spec.ControllerVersion)
-	asset, err := DetermineGithubAsset(f.Spec.ControllerVersion)
+	_, err := url.Parse(f.Spec.ControllerURL)
 	if err != nil {
 		return err
 	}
 
-	binaryReader, checksum, err := FetchGithubAssetAndChecksum(asset)
+	recorder.Eventf(&f, corev1.EventTypeNormal, "Self-Reconcilation", "replacing firewall-controller version %s with version %s", v.Version, f.Spec.ControllerVersion)
+
+	binaryReader, checksum, err := FetchBinaryAndChecksum(f.Spec.ControllerURL)
 	if err != nil {
-		return fmt.Errorf("could not fetch github asset and checksum for firewall-controller version %s, err: %w", f.Spec.ControllerVersion, err)
+		return fmt.Errorf("could not download binary or checksum for firewall-controller version %s, err: %w", f.Spec.ControllerVersion, err)
 	}
 
 	err = replaceBinary(binaryReader, checksum)
@@ -61,49 +58,17 @@ func UpdateToSpecVersion(f firewallv1.Firewall, log logr.Logger, recorder record
 	return nil
 }
 
-func DetermineGithubAsset(githubTag string) (*github.ReleaseAsset, error) {
-	client := github.NewClient(nil)
-	releases, _, err := client.Repositories.ListReleases(context.Background(), gitHubOwner, gitHubRepo, &github.ListOptions{})
+func FetchBinaryAndChecksum(url string) (io.ReadCloser, string, error) {
+	checksum, err := slurpFile(url + ".sha256")
 	if err != nil {
-		panic(err)
-	}
-
-	var rel *github.RepositoryRelease
-	for _, r := range releases {
-		if r.TagName != nil && *r.TagName == githubTag {
-			rel = r
-			break
-		}
-	}
-
-	if rel == nil {
-		return nil, fmt.Errorf("could not find release with tag %s", githubTag)
+		return nil, "", fmt.Errorf("could not slurp checksum file at %s, err: %w", url, err)
 	}
 
-	var asset *github.ReleaseAsset
-	for _, ra := range rel.Assets {
-		if ra.GetName() == gitHubArtifact {
-			asset = &ra
-			break
-		}
-	}
-
-	if asset == nil {
-		return nil, fmt.Errorf("could not find artifact %s in github release with tag %s", gitHubArtifact, githubTag)
-	}
-	return asset, nil
-}
-
-func FetchGithubAssetAndChecksum(ra *github.ReleaseAsset) (io.ReadCloser, string, error) {
-	checksum, err := slurpFile(ra.GetBrowserDownloadURL() + ".sha256")
+	resp, err := http.Get(url)
 	if err != nil {
-		return nil, "", fmt.Errorf("could not slurp checksum file for asset %s, err: %w", ra.GetBrowserDownloadURL(), err)
+		return nil, "", fmt.Errorf("could not download url %s, err: %w", url, err)
 	}
 
-	resp, err := http.Get(ra.GetBrowserDownloadURL())
-	if err != nil {
-		return nil, "", fmt.Errorf("could not download asset %s, err: %w", ra.GetBrowserDownloadURL(), err)
-	}
 	return resp.Body, checksum, nil
 }