Add test coverage to workflows

This change adds code coverage to our workflows and the necessary
Makefile targets to support this.

Signed-off-by: David Son <[email protected]>

.github/workflows/build.yml

@@ -70,7 +70,9 @@ jobs:
         if: matrix.os == 'al2-arm'
         run: dnf install zlib-static.aarch64 -y
       - run: make
-      - run: make test
+      - run: make test-with-coverage
+      - name: Show test coverage
+        run: make show-test-coverage
 
   integration:
     needs: setup
@@ -91,4 +93,6 @@ jobs:
       - name: Install zlib static on AL2 ARM instances
         if: matrix.os == 'al2-arm'
         run: dnf install zlib-static.aarch64 -y
-      - run: make integration
+      - run: make integration-with-coverage
+      - name: Show test coverage
+        run: make show-integration-coverage

.gitignore

@@ -1,3 +1,4 @@
+/cov
 /out
 /release
 *.db

Dockerfile

@@ -24,9 +24,11 @@ ARG CONTAINERD_VERSION
 ARG RUNC_VERSION
 ARG NERDCTL_VERSION
 ARG TARGETARCH
+ENV GOPROXY direct
+ENV GOCOVERDIR /test_coverage
+
 COPY ./integ_entrypoint.sh /integ_entrypoint.sh
 COPY . $GOPATH/src/github.com/awslabs/soci-snapshotter
-ENV GOPROXY direct
 RUN apk update && apk upgrade
 RUN apk add --no-cache \
     btrfs-progs-libs \

Makefile

@@ -17,6 +17,7 @@
 CMD_DESTDIR ?= /usr/local
 GO111MODULE_VALUE=auto
 OUTDIR ?= $(CURDIR)/out
+COVDIR ?= $(CURDIR)/cov
 PKG=github.com/awslabs/soci-snapshotter
 VERSION=$(shell git describe --match 'v[0-9]*' --dirty='.m' --always --tags)
 REVISION=$(shell git rev-parse HEAD)$(shell if ! git diff --no-ext-diff --quiet --exit-code; then echo .m; fi)
@@ -55,11 +56,18 @@ CMD=soci-snapshotter-grpc soci
 
 CMD_BINARIES=$(addprefix $(OUTDIR)/,$(CMD))
 
+PACKAGE_LIST_CMD=go list -f '{{.ImportPath}}' ./... | grep -v "benchmark" | paste -sd ","
+
+SOCI_LIBRARY_PACKAGE_LIST=$(shell $(PACKAGE_LIST_CMD))
+SOCI_CLI_PACKAGE_LIST=$(shell echo $(SOCI_LIBRARY_PACKAGE_LIST),$(shell cd $(SOCI_SNAPSHOTTER_PROJECT_ROOT)/cmd/soci && $(PACKAGE_LIST_CMD)))
+SOCI_GRPC_PACKAGE_LIST=$(shell echo $(SOCI_LIBRARY_PACKAGE_LIST),$(shell cd $(SOCI_SNAPSHOTTER_PROJECT_ROOT)/cmd/soci-snapshotter-grpc && $(PACKAGE_LIST_CMD)))
+
 GO_BENCHMARK_TESTS?=.
 
-.PHONY: all build check flatc add-ltag install uninstall tidy vendor clean \
-	clean-integration test integration release benchmarks build-benchmarks \
-	benchmarks-perf-test benchmarks-comparison-test
+.PHONY: all build check flatc add-ltag install uninstall tidy vendor clean clean-coverage \
+	clean-integration test test-with-coverage show-test-coverage show-test-coverage-html \
+	integration integration-with-coverage show-integration-coverage show-integration-coverage-html \
+	release benchmarks build-benchmarks benchmarks-perf-test benchmarks-comparison-test
 
 all: build
 
@@ -95,12 +103,15 @@ uninstall:
 	@echo "$@"
 	@rm -f $(addprefix $(CMD_DESTDIR)/bin/,$(notdir $(CMD_BINARIES)))
 
-clean: clean-integration
+clean: clean-integration clean-coverage
 	@echo "🧹 ... 🗑️"
 	@rm -rf $(OUTDIR)
 	@rm -rf $(CURDIR)/release/
 	@echo "All clean!"
 
+clean-coverage:
+	@rm -rf $(COVDIR)
+
 clean-integration:
 	@echo "🧹 Cleaning leftover integration test artifacts..."
 
@@ -127,13 +138,50 @@ vendor:
 
 test:
 	@echo "$@"
-	@GO111MODULE=$(GO111MODULE_VALUE) go test $(GO_TEST_FLAGS) $(GO_LD_FLAGS) -race ./...
+	@GO111MODULE=$(GO111MODULE_VALUE) go test $(GO_TEST_FLAGS) $(GO_LD_FLAGS) -race `go list ./... | grep -v benchmark` -args $(GO_TEST_ARGS)
+
+show-test-coverage: test-with-coverage
+	go tool covdata percent -i $(COVDIR)/unit
+
+show-test-coverage-html: test-with-coverage $(COVDIR)/html
+	go tool covdata textfmt -i $(COVDIR)/unit -o $(COVDIR)/unit/unit.out
+	go tool cover -html=$(COVDIR)/unit/unit.out -o $(COVDIR)/html/unit.html
+
+test-with-coverage: $(COVDIR)/unit
+
+$(COVDIR):
+	@mkdir -p $@
+
+$(COVDIR)/html:
+	@mkdir -p $@
+
+$(COVDIR)/unit: $(COVDIR)
+	@mkdir -p $@
+	GO_TEST_FLAGS="$(GO_TEST_FLAGS) -cover" \
+	GO_TEST_ARGS="-test.gocoverdir=$(COVDIR)/unit" \
+	GO_BUILD_FLAGS="$(GO_BUILD_FLAGS) -coverpkg=$(SOCI_LIBRARY_PACKAGE_LIST)"\
+		$(MAKE) test
 
 integration: build
 	@echo "$@"
 	@echo "SOCI_SNAPSHOTTER_PROJECT_ROOT=$(SOCI_SNAPSHOTTER_PROJECT_ROOT)"
 	@GO111MODULE=$(GO111MODULE_VALUE) SOCI_SNAPSHOTTER_PROJECT_ROOT=$(SOCI_SNAPSHOTTER_PROJECT_ROOT) ENABLE_INTEGRATION_TEST=true go test $(GO_TEST_FLAGS) -v -timeout=0 ./integration
 
+show-integration-coverage: integration-with-coverage
+	go tool covdata percent -i $(COVDIR)/integration
+
+show-integration-coverage-html: integration-with-coverage $(COVDIR)/html
+	go tool covdata textfmt -i $(COVDIR)/integration -o $(COVDIR)/integration/integration.out
+	go tool cover -html=$(COVDIR)/integration/integration.out -o $(COVDIR)/html/integration.html
+
+integration-with-coverage: $(COVDIR)/integration
+
+$(COVDIR)/integration: $(COVDIR)
+	@mkdir -p $@
+	GO_TEST_FLAGS="$(GO_TEST_FLAGS)" \
+	GO_BUILD_FLAGS="$(GO_BUILD_FLAGS) -coverpkg=$(SOCI_CLI_PACKAGE_LIST),$(SOCI_GRPC_PACKAGE_LIST)" \
+		$(MAKE) integration
+
 release:
 	@echo "$@"
 	@$(SOCI_SNAPSHOTTER_PROJECT_ROOT)/scripts/create-releases.sh $(RELEASE_TAG)

integration/util_test.go

@@ -41,6 +41,7 @@ import (
 	"encoding/csv"
 	"encoding/json"
 	"encoding/pem"
+	"errors"
 	"fmt"
 	"math/big"
 	"os"
@@ -148,12 +149,14 @@ services:
    entrypoint: [ "/integ_entrypoint.sh" ]
    environment:
     - NO_PROXY=127.0.0.1,localhost
+    - GOCOVERDIR=/test_coverage
    tmpfs:
     - /tmp:exec,mode=777
     - /var/lib/containerd
     - /var/lib/soci-snapshotter-grpc
    volumes:
     - /dev/fuse:/dev/fuse
+    - {{.ImageContextDir}}/cov/integration:/test_coverage
 `
 const composeRegistryTemplate = `
 version: "3.7"
@@ -165,12 +168,14 @@ services:
   entrypoint: [ "/integ_entrypoint.sh" ]
   environment:
    - NO_PROXY=127.0.0.1,localhost,{{.RegistryHost}}:443
+   - GOCOVERDIR=/test_coverage
   tmpfs:
    - /tmp:exec,mode=777
    - /var/lib/containerd
    - /var/lib/soci-snapshotter-grpc
   volumes:
    - /dev/fuse:/dev/fuse
+   - {{.ImageContextDir}}/cov/integration:/test_coverage
  registry:
   image: {{.RegistryImageRef}}
   container_name: {{.RegistryHost}}
@@ -197,12 +202,14 @@ services:
     entrypoint: [ "/integ_entrypoint.sh" ]
     environment:
     - NO_PROXY=127.0.0.1,localhost,{{.RegistryHost}}:443
+    - GOCOVERDIR=/test_coverage
     tmpfs:
     - /tmp:exec,mode=777
     - /var/lib/containerd
     - /var/lib/soci-snapshotter-grpc
     volumes:
     - /dev/fuse:/dev/fuse
+    - {{.ImageContextDir}}/cov/integration:/test_coverage
   registry:
     image: {{.RegistryImageRef}}
     container_name: {{.RegistryHost}}
@@ -473,6 +480,11 @@ func newShellWithRegistry(t *testing.T, r registryConfig, opts ...registryOpt) (
 		serviceName = "testing"
 	)
 
+	pRoot, err := testutil.GetProjectRoot()
+	if err != nil {
+		t.Fatal(err)
+	}
+
 	// Setup dummy creds for test
 	crt, key, err := generateRegistrySelfSignedCert(r.host)
 	if err != nil {
@@ -541,6 +553,7 @@ networks:
 
 	s, err := testutil.ApplyTextTemplate(composeRegistryTemplate, dockerComposeYaml{
 		ServiceName:      serviceName,
+		ImageContextDir:  pRoot,
 		RegistryHost:     r.host,
 		RegistryImageRef: rOpts.registryImageRef,
 		HostVolumeMount:  hostVolumeMount,
@@ -567,12 +580,13 @@ networks:
 		X("update-ca-certificates").
 		Retry(100, "nerdctl", "login", "-u", r.user, "-p", r.pass, r.host)
 	return sh, func() error {
-		if err := c.Cleanup(); err != nil {
-			return err
+		killErr := testutil.KillMatchingProcess(sh, "soci-snapshotter-grpc")
+		if err = c.Cleanup(); err != nil {
+			return errors.Join(killErr, err)
 		}
 		for _, f := range cleanups {
 			if err := f(); err != nil {
-				return err
+				return errors.Join(killErr, err)
 			}
 		}
 		return nil
@@ -585,7 +599,18 @@ func newSnapshotterBaseShell(t *testing.T) (*shell.Shell, func() error) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	c, err := compose.Up(composeDefaultTemplate, compose.WithBuildArgs(buildArgs...), compose.WithStdio(testutil.TestingLogDest()))
+
+	pRoot, err := testutil.GetProjectRoot()
+	if err != nil {
+		t.Fatal(err)
+	}
+	s, err := testutil.ApplyTextTemplate(composeDefaultTemplate, dockerComposeYaml{
+		ImageContextDir: pRoot,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	c, err := compose.Up(s, compose.WithBuildArgs(buildArgs...), compose.WithStdio(testutil.TestingLogDest()))
 	if err != nil {
 		t.Fatalf("failed to prepare compose: %v", err)
 	}
@@ -599,7 +624,14 @@ func newSnapshotterBaseShell(t *testing.T) (*shell.Shell, func() error) {
 			t.Fatalf("failed to write containerd config %v: %v", defaultContainerdConfigPath, err)
 		}
 	}
-	return sh, c.Cleanup
+	cleanup := func() error {
+		err := testutil.KillMatchingProcess(sh, "soci-snapshotter-grpc")
+		if err != nil {
+			return fmt.Errorf("stop-soci-err: %v, compose-cleanup-err: %v", err, c.Cleanup())
+		}
+		return c.Cleanup()
+	}
+	return sh, cleanup
 }
 
 func generateRegistrySelfSignedCert(registryHost string) (crt, key []byte, _ error) {

util/testutil/shell.go

@@ -367,7 +367,8 @@ func KillMatchingProcess(sh *shell.Shell, psLinePattern string) error {
 
 	var allErr error
 	for _, pid := range targets {
-		if err := sh.Command("kill", "-9", fmt.Sprintf("%d", pid)).Run(); err != nil {
+		// Send SIGTERM so the unit under test correctly writes integration coverage reports to Go coverage directory.
+		if err := sh.Command("kill", "-2", fmt.Sprintf("%d", pid)).Run(); err != nil {
 			errors.Join(allErr, fmt.Errorf("failed to kill %v: %w", pid, err))
 		}
 	}