Add integration tests and support for pushing static file/RPM in a st…

…ream fashion. Fix beskar registry transport issue.

.github/workflows/ci.yml

@@ -9,18 +9,31 @@ jobs:
     steps:
       - uses: actions/setup-go@v3
         with:
-          go-version: '1.20'
+          go-version: '1.21'
       - uses: actions/checkout@v3
       - name: Run linters
         run: ./scripts/mage lint:all
 
+  tests:
+    name: tests
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/setup-go@v3
+        with:
+          go-version: '1.21'
+      - uses: actions/checkout@v3
+      - name: Run unit tests
+        run: ./scripts/mage test:unit
+      - name: Run integration tests
+        run: ./scripts/mage test:integration
+
   build:
     name: build
     runs-on: ubuntu-22.04
     steps:
       - uses: actions/setup-go@v3
         with:
-          go-version: '1.20'
+          go-version: '1.21'
       - uses: actions/checkout@v3
       - name: Build binaries
         run: ./scripts/mage build:all

.github/workflows/release.yml

@@ -11,19 +11,32 @@ jobs:
     steps:
       - uses: actions/setup-go@v3
         with:
-          go-version: '1.20'
+          go-version: '1.21'
       - uses: actions/checkout@v3
       - name: Run linters
         run: ./scripts/mage lint:all
 
+  tests:
+    name: tests
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/setup-go@v3
+        with:
+          go-version: '1.21'
+      - uses: actions/checkout@v3
+      - name: Run unit tests
+        run: ./scripts/mage test:unit
+      - name: Run integration tests
+        run: ./scripts/mage test:integration
+
   release-beskar:
     name: release beskar
-    needs: lint
+    needs: [lint, tests]
     runs-on: ubuntu-22.04
     steps:
       - uses: actions/setup-go@v3
         with:
-          go-version: '1.20'
+          go-version: '1.21'
       - uses: actions/checkout@v3
       - name: Release beskar image
         run: ./scripts/mage ci:image ghcr.io/ctrliq/beskar:${{ github.ref_name }} "${{ github.actor }}" "${{ secrets.GITHUB_TOKEN }}"
@@ -32,12 +45,12 @@ jobs:
 
   release-beskar-yum:
     name: release beskar-yum
-    needs: lint
+    needs: [lint, tests]
     runs-on: ubuntu-22.04
     steps:
       - uses: actions/setup-go@v3
         with:
-          go-version: '1.20'
+          go-version: '1.21'
       - uses: actions/checkout@v3
       - name: Release beskar-yum image
         run: ./scripts/mage ci:image ghcr.io/ctrliq/beskar-yum:${{ github.ref_name }} "${{ github.actor }}" "${{ secrets.GITHUB_TOKEN }}"
@@ -46,12 +59,12 @@ jobs:
 
   release-beskar-static:
     name: release beskar-static
-    needs: lint
+    needs: [lint, tests]
     runs-on: ubuntu-22.04
     steps:
       - uses: actions/setup-go@v3
         with:
-          go-version: '1.20'
+          go-version: '1.21'
       - uses: actions/checkout@v3
       - name: Release beskar-static image
         run: ./scripts/mage ci:image ghcr.io/ctrliq/beskar-static:${{ github.ref_name }} "${{ github.actor }}" "${{ secrets.GITHUB_TOKEN }}"

build/mage/build.go

@@ -54,44 +54,64 @@ type binaryConfig struct {
 	genAPI            *genAPI
 	buildTags         []string
 	baseImage         string
+	integrationTest   *integrationTest
 }
 
 const (
-	beskarBinary       = "beskar"
-	beskarctlBinary    = "beskarctl"
-	beskarYUMBinary    = "beskar-yum"
-	beskarStaticBinary = "beskar-static"
+	BeskarBinary       = "beskar"
+	BeskarctlBinary    = "beskarctl"
+	BeskarYUMBinary    = "beskar-yum"
+	BeskarStaticBinary = "beskar-static"
 )
 
 var binaries = map[string]binaryConfig{
-	beskarBinary: {
+	BeskarBinary: {
 		configFiles: map[string]string{
 			"internal/pkg/config/default/beskar.yaml": "/etc/beskar/beskar.yaml",
 		},
 		useProto:  true,
 		buildTags: []string{"include_gcs"},
+		integrationTest: &integrationTest{
+			envs: map[string]string{
+				"BESKAR_REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY": "/tmp/integration/registry",
+				"BESKAR_REGISTRY_LOG_ACCESSLOG_DISABLED":           "true",
+				"BESKAR_REGISTRY_HTTP_ADDR":                        "127.0.0.1:5100",
+				"BESKAR_GOSSIP_ADDR":                               "127.0.0.1:5102",
+				"BESKAR_CACHE_ADDR":                                "127.0.0.1:5103",
+			},
+		},
 	},
-	beskarctlBinary: {},
-	beskarYUMBinary: {
+	BeskarctlBinary: {},
+	BeskarYUMBinary: {
 		configFiles: map[string]string{
 			"internal/plugins/yum/pkg/config/default/beskar-yum.yaml": "/etc/beskar/beskar-yum.yaml",
 		},
 		execStmts: [][]string{
 			{
-				//"apk", "add", "-U", "bash", "--repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing/",
-				"sh", "-c", "apt-get update -y && apt-get install -y --no-install-recommends ca-certificates createrepo-c && " +
-					"rm -rf /var/lib/apt/lists/* && rm -Rf /usr/share/doc && rm -Rf /usr/share/man && apt-get clean",
+				"apk", "add", "createrepo_c", "--repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing/",
+				// NOTE: restore in case alpine createrepo_c package is broken again
+				//"sh", "-c", "apt-get update -y && apt-get install -y --no-install-recommends ca-certificates createrepo-c && " +
+				//	"rm -rf /var/lib/apt/lists/* && rm -Rf /usr/share/doc && rm -Rf /usr/share/man && apt-get clean",
 			},
 		},
 		genAPI: &genAPI{
 			path:          "pkg/plugins/yum/api/v1",
 			filename:      "api.go",
 			interfaceName: "YUM",
 		},
-		useProto:  true,
-		baseImage: "debian:bullseye-slim",
+		useProto: true,
+		// NOTE: restore in case alpine createrepo_c package is broken again
+		//baseImage: "debian:bullseye-slim",
+		integrationTest: &integrationTest{
+			isPlugin: true,
+			envs: map[string]string{
+				"BESKARYUM_ADDR":                         "127.0.0.1:5200",
+				"BESKARYUM_GOSSIP_ADDR":                  "127.0.0.1:5201",
+				"BESKARYUM_STORAGE_FILESYSTEM_DIRECTORY": "/tmp/integration/beskar-yum",
+			},
+		},
 	},
-	beskarStaticBinary: {
+	BeskarStaticBinary: {
 		configFiles: map[string]string{
 			"internal/plugins/static/pkg/config/default/beskar-static.yaml": "/etc/beskar/beskar-static.yaml",
 		},
@@ -100,8 +120,15 @@ var binaries = map[string]binaryConfig{
 			filename:      "api.go",
 			interfaceName: "Static",
 		},
-		useProto:  true,
-		baseImage: BaseImage,
+		useProto: true,
+		integrationTest: &integrationTest{
+			isPlugin: true,
+			envs: map[string]string{
+				"BESKARSTATIC_ADDR":                         "127.0.0.1:5300",
+				"BESKARSTATIC_GOSSIP_ADDR":                  "127.0.0.1:5301",
+				"BESKARSTATIC_STORAGE_FILESYSTEM_DIRECTORY": "/tmp/integration/beskar-static",
+			},
+		},
 	},
 }
 
@@ -131,18 +158,18 @@ func (b Build) Proto(ctx context.Context) error {
 }
 
 func (b Build) Beskar(ctx context.Context) error {
-	return b.build(ctx, beskarBinary)
+	return b.build(ctx, BeskarBinary)
 }
 
 func (b Build) Beskarctl(ctx context.Context) error {
-	return b.build(ctx, beskarctlBinary)
+	return b.build(ctx, BeskarctlBinary)
 }
 
 func (b Build) Plugins(ctx context.Context) {
 	mg.CtxDeps(
 		ctx,
-		mg.F(b.Plugin, beskarYUMBinary),
-		mg.F(b.Plugin, beskarStaticBinary),
+		mg.F(b.Plugin, BeskarYUMBinary),
+		mg.F(b.Plugin, BeskarStaticBinary),
 	)
 }
 
@@ -172,13 +199,13 @@ func (b Build) build(ctx context.Context, name string) error {
 		}
 	}
 
+	buildOpts, ok := getBuildOptions(ctx)
+
 	currentPlatform, err := getCurrentPlatform()
 	if err != nil {
 		return err
 	}
 
-	buildOpts, ok := getBuildOptions(ctx)
-
 	if !ok {
 		buildOpts = &buildOptions{
 			platforms: []dagger.Platform{
@@ -191,6 +218,17 @@ func (b Build) build(ctx context.Context, name string) error {
 		}
 	}
 
+	files, err := getGoFiles(filepath.Join("cmd", name))
+	if err != nil {
+		return err
+	}
+	changed, err := target.Path(filepath.Join("build/output", name), files...)
+	if err != nil {
+		return err
+	} else if !changed && !ok {
+		return nil
+	}
+
 	client := buildOpts.client
 
 	if client == nil {

build/mage/common.go

@@ -10,14 +10,15 @@ import (
 
 	"dagger.io/dagger"
 	"golang.org/x/sys/cpu"
+	"golang.org/x/tools/go/packages"
 )
 
 const (
-	GoImage           = "golang:1.21.4-alpine"
+	GoImage           = "golang:1.21.5-alpine3.19"
 	GolangCILintImage = "golangci/golangci-lint:v1.54.2-alpine"
 	HelmImage         = "alpine/helm:3.12.2"
 	ProtolintImage    = "yoheimuta/protolint:0.45.0"
-	BaseImage         = "alpine:3.17"
+	BaseImage         = "alpine:3.19"
 
 	ProtocVersion    = "v23.4"
 	ProtocFileFormat = "protoc-23.4-linux-%s.zip"
@@ -65,10 +66,28 @@ func getSource(client *dagger.Client) *dagger.Directory {
 			"README.md",
 			"go.work",
 			"go.work.sum",
+			"integration",
 		},
 	})
 }
 
+func getIntegrationSource(client *dagger.Client) *dagger.Directory {
+	return client.Host().Directory(".", dagger.HostDirectoryOpts{
+		Include: []string{
+			"go.mod",
+			"go.sum",
+			"integration",
+			"pkg",
+		},
+	})
+}
+
+func getBuildBinaries(client *dagger.Client, binaries ...string) *dagger.Directory {
+	return client.Host().Directory("build/output", dagger.HostDirectoryOpts{
+		Include: binaries,
+	})
+}
+
 func getProtoSource(client *dagger.Client) *dagger.Directory {
 	return client.Host().Directory("api")
 }
@@ -122,3 +141,48 @@ func getPlatformBinarySuffix(platform string) string {
 	platform = strings.TrimPrefix(platform, "linux/")
 	return strings.ReplaceAll(platform, "/", "-")
 }
+
+func getGoFiles(dir string) ([]string, error) {
+	cfg := packages.Config{
+		Mode: packages.NeedFiles |
+			packages.NeedEmbedFiles |
+			packages.NeedImports |
+			packages.NeedName |
+			packages.NeedDeps |
+			packages.NeedModule,
+		Dir: dir,
+	}
+
+	initial, err := packages.Load(&cfg, ".")
+	if err != nil {
+		return nil, fmt.Errorf("while loading directory %s: %s", dir, err)
+	}
+	for _, pkg := range initial {
+		if len(pkg.Errors) > 0 {
+			return nil, fmt.Errorf("package error: %s", pkg.Errors[0])
+		}
+	}
+
+	fileMap := make(map[string]struct{})
+	files := make([]string, 0)
+
+	add := func(sourceFiles []string) {
+		for _, file := range sourceFiles {
+			if _, ok := fileMap[file]; !ok {
+				fileMap[file] = struct{}{}
+				files = append(files, file)
+			}
+		}
+	}
+
+	packages.Visit(initial, nil, func(pkg *packages.Package) {
+		if pkg.Module == nil {
+			return
+		}
+		add(pkg.GoFiles)
+		add(pkg.OtherFiles)
+		add(pkg.EmbedFiles)
+	})
+
+	return files, nil
+}

build/mage/go.mod

@@ -1,21 +1,24 @@
 module go.ciq.dev/beskar/build/mage
 
-go 1.20
+go 1.21
+
+toolchain go1.21.0
 
 require (
-	dagger.io/dagger v0.7.4
+	dagger.io/dagger v0.9.4
 	github.com/magefile/mage v1.15.0
-	golang.org/x/sys v0.12.0
+	golang.org/x/sys v0.14.0
+	golang.org/x/tools v0.14.0
 )
 
 require (
 	github.com/99designs/gqlgen v0.17.31 // indirect
 	github.com/Khan/genqlient v0.6.0 // indirect
 	github.com/adrg/xdg v0.4.0 // indirect
-	github.com/iancoleman/strcase v0.3.0 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/stretchr/testify v1.8.4 // indirect
 	github.com/vektah/gqlparser/v2 v2.5.6 // indirect
-	golang.org/x/mod v0.12.0 // indirect
-	golang.org/x/sync v0.3.0 // indirect
-	golang.org/x/tools v0.13.0 // indirect
+	golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
+	golang.org/x/mod v0.13.0 // indirect
+	golang.org/x/sync v0.4.0 // indirect
 )