ci(disk usage): disk related fixes and improvements (#1524)

ci(workflow): show disk usage and free up disk space used by unneeded tooling ci(tests): routes tests: do not copy large images if they are not used later ci(trivy): update a test: download trivy.db to a temporary folder Signed-off-by: Andrei Aaron <[email protected]>
project-zot · Jun 15, 2023 · f9f9422 · f9f9422
1 parent 988ca42
commit f9f9422
Show file tree

Hide file tree

Showing 4 changed files with 52 additions and 7 deletions.
diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml
@@ -32,14 +32,21 @@ jobs:
         os: [linux, darwin]
         arch: [amd64, arm64]
     steps:
+      - name: Remove unneeded tooling
+        run: |
+          # To free up ~15 GB of disk space
+          sudo rm -rf /opt/ghc
+          sudo rm -rf /usr/local/share/boost
+          sudo rm -rf /usr/local/lib/android
+          sudo rm -rf /usr/share/dotnet
       - name: Install go
         uses: actions/setup-go@v4
         with:
           cache: false
           go-version: 1.20.x
+
       - name: Check out source code
         uses: actions/checkout@v3
-
       - name: Cache go dependencies
         id: cache-go-dependencies
         uses: actions/cache@v3
@@ -94,6 +101,23 @@ jobs:
         env:
           AWS_ACCESS_KEY_ID: fake
           AWS_SECRET_ACCESS_KEY: fake
+
+      - name: Check disk space before build
+        run: |
+          cd $GITHUB_WORKSPACE
+          set -x
+          df -h
+          sudo ls -lRh /tmp/* || true
+          sudo du -sh /tmp || true
+          sudo du -sh /tmp/* || true
+          sudo find /tmp/ -size +5M | sudo xargs ls -lh
+          du -sh ./* || true
+          find ./ -size +5M   | xargs ls -lh
+          sudo du -sh /var/
+          sudo du -sh /var/lib/docker/
+          du -sh /home/runner/work/
+          set +x
+
       - name: Run build and test
         timeout-minutes: 80
         run: |
@@ -112,6 +136,24 @@ jobs:
           AWS_SECRET_ACCESS_KEY: fake
           OS: ${{ matrix.os }}
           ARCH: ${{ matrix.arch }}
+
+      - name: Check disk space after build
+        if: always()
+        run: |
+          cd $GITHUB_WORKSPACE
+          set -x
+          df -h
+          sudo ls -lRh /tmp/* || true
+          sudo du -sh /tmp || true
+          sudo du -sh /tmp/* || true
+          sudo find /tmp/ -size +5M | sudo xargs ls -lh
+          du -sh ./* || true
+          find ./ -size +5M   | xargs ls -lh
+          sudo du -sh /var/
+          sudo du -sh /var/lib/docker/
+          du -sh /home/runner/work/
+          set +x
+
       - name: Upload code coverage
         uses: codecov/codecov-action@v3
         with:

diff --git a/Makefile b/Makefile
@@ -106,10 +106,14 @@ exporter-minimal: modcheck build-metadata
 test: $(if $(findstring ui,$(BUILD_LABELS)), ui)
 test: check-skopeo $(TESTDATA) $(ORAS)
 	go test -failfast -tags $(BUILD_LABELS),containers_image_openpgp -v -trimpath -race -timeout 15m -cover -coverpkg ./... -coverprofile=coverage-extended.txt -covermode=atomic ./...
+	rm -rf /tmp/getter*; rm -rf /tmp/trivy*
 	go test -failfast -tags containers_image_openpgp -v -trimpath -race -cover -coverpkg ./... -coverprofile=coverage-minimal.txt -covermode=atomic ./...
+	rm -rf /tmp/getter*; rm -rf /tmp/trivy*
 	# development-mode unit tests possibly using failure injection
 	go test -failfast -tags dev,$(BUILD_LABELS),containers_image_openpgp -v -trimpath -race -timeout 15m -cover -coverpkg ./... -coverprofile=coverage-dev-extended.txt -covermode=atomic ./pkg/test/... ./pkg/api/... ./pkg/storage/... ./pkg/extensions/sync/... -run ^TestInject
+	rm -rf /tmp/getter*; rm -rf /tmp/trivy*
 	go test -failfast -tags dev,containers_image_openpgp -v -trimpath -race -cover -coverpkg ./... -coverprofile=coverage-dev-minimal.txt -covermode=atomic ./pkg/test/... ./pkg/storage/... ./pkg/extensions/sync/... -run ^TestInject
+	rm -rf /tmp/getter*; rm -rf /tmp/trivy*
 	go test -failfast -tags stress,$(BUILD_LABELS),containers_image_openpgp -v -trimpath -race -timeout 15m ./pkg/cli/stress_test.go
 
 .PHONY: privileged-test

diff --git a/pkg/api/routes_test.go b/pkg/api/routes_test.go
@@ -41,11 +41,6 @@ func TestRoutes(t *testing.T) {
 		ctlr.Config.Storage.RootDirectory = t.TempDir()
 		ctlr.Config.Storage.Commit = true
 
-		err := test.CopyFiles("../../test/data", ctlr.Config.Storage.RootDirectory)
-		if err != nil {
-			panic(err)
-		}
-
 		cm := test.NewControllerManager(ctlr)
 		cm.StartAndWait(port)
 		defer cm.StopServer()

diff --git a/pkg/extensions/extension_search_test.go b/pkg/extensions/extension_search_test.go
@@ -43,7 +43,11 @@ func TestTrivyDBGenerator(t *testing.T) {
 			},
 		}
 		storeController := storage.StoreController{
-			DefaultStore: mocks.MockedImageStore{},
+			DefaultStore: mocks.MockedImageStore{
+				RootDirFn: func() string {
+					return t.TempDir()
+				},
+			},
 		}
 
 		cveInfo := cveinfo.NewCVEInfo(storeController, repoDB, "ghcr.io/project-zot/trivy-db", "", logger)