fix(ci): fix nighlty builds and print zot log on failure (#1799)

now gc stress on s3 storage is using minio for ci/cd builds gc stress on s3 storage is using localstack for nightly builds fixed(gc): make sure we don't remove repo if there are blobs being uploaded or the number of blobs gc'ed is not 0 Signed-off-by: Petu Eusebiu <[email protected]>
project-zot · Sep 20, 2023 · f164fb9 · f164fb9
1 parent a11fe2d
commit f164fb9
Show file tree

Hide file tree

Showing 10 changed files with 331 additions and 39 deletions.
diff --git a/.github/workflows/gc-stress-test.yaml b/.github/workflows/gc-stress-test.yaml
@@ -24,6 +24,7 @@ jobs:
           go-version: 1.20.x
 
       - name: Run zb
+        id: bench
         run: |
             make binary
             make bench
@@ -35,6 +36,13 @@ jobs:
 
             # clean zot storage
             sudo rm -rf /tmp/zot
+        continue-on-error: true
+
+      - name: Check on failures
+        if: steps.bench.outcome != 'success'
+        run: |
+          cat /tmp/gc-referrers-bench-local.log
+          exit 1
 
   gc-stress-local:
     name: GC(without referrers) on filesystem with short interval
@@ -48,6 +56,7 @@ jobs:
           go-version: 1.20.x
 
       - name: Run zb
+        id: bench
         run: |
             make binary
             make bench
@@ -59,3 +68,188 @@ jobs:
 
             # clean zot storage
             sudo rm -rf /tmp/zot
+        continue-on-error: true
+
+      - name: Check on failures
+        if: steps.bench.outcome != 'success'
+        run: |
+          cat /tmp/gc-bench-local.log
+          exit 1
+          
+  gc-referrers-stress-s3:
+    name: GC(with referrers) on S3(minio) with short interval
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/clean-runner
+      - uses: actions/setup-go@v4
+        with:
+          cache: false
+          go-version: 1.20.x
+      - name: Setup localstack service
+        run: |
+            pip install localstack                    # Install LocalStack cli
+            docker pull localstack/localstack:1.3     # Make sure to pull the latest version of the image
+            localstack start -d                       # Start LocalStack in the background
+            
+            echo "Waiting for LocalStack startup..."  # Wait 30 seconds for the LocalStack container
+            localstack wait -t 30                     # to become ready before timing out 
+            echo "Startup complete"
+            
+            # aws --endpoint-url=http://localhost:4566 s3api create-bucket --bucket zot-storage --region us-east-2 --create-bucket-configuration="{\"LocationConstraint\": \"us-east-2\"}"
+            aws dynamodb --endpoint-url http://localhost:4566 --region "us-east-2" create-table --table-name BlobTable --attribute-definitions AttributeName=Digest,AttributeType=S --key-schema AttributeName=Digest,KeyType=HASH --provisioned-throughput ReadCapacityUnits=10,WriteCapacityUnits=5
+        env:
+          AWS_ACCESS_KEY_ID: fake
+          AWS_SECRET_ACCESS_KEY: fake
+      - name: Setup minio service
+        run: |
+            docker run -d -p 9000:9000 --name minio \
+                -e "MINIO_ACCESS_KEY=minioadmin" \
+                -e "MINIO_SECRET_KEY=minioadmin" \
+                -v /tmp/data:/data \
+                -v /tmp/config:/root/.minio \
+                --health-cmd "curl http://localhost:9000/minio/health/live" \
+                minio/minio:edge-cicd server /data
+      - name: Install py minio
+        run: pip3 install minio
+
+      - name: Wait for minio to come up
+        run: |
+          sleep 10
+          curl --connect-timeout 5 \
+            --max-time 120 \
+            --retry 12 \
+            --retry-max-time 120 \
+            'http://localhost:9000/minio/health/live'
+
+      - name: Create minio bucket
+        run: |
+            python3 - <<'EOF'
+            from minio import Minio
+
+            try:
+                minio = Minio(
+                    'localhost:9000',
+                    access_key='minioadmin',
+                    secret_key='minioadmin',
+                    secure=False
+                )
+            except Exception as ex:
+                raise
+
+            minio.make_bucket('zot-storage')
+            print(f'{minio.list_buckets()}')
+            EOF
+
+      - name: Run zb
+        id: bench
+        run: |
+            make binary
+            make bench
+            ./bin/zot-linux-amd64 serve test/gc-stress/config-gc-referrers-bench-s3-minio.json &
+            sleep 10
+            bin/zb-linux-amd64 -c 10 -n 100 -o ci-cd http://localhost:8080
+
+            killall -r zot-*
+
+            # clean zot storage
+            sudo rm -rf /tmp/zot
+        env:
+          AWS_ACCESS_KEY_ID: fake
+          AWS_SECRET_ACCESS_KEY: fake
+        continue-on-error: true
+
+      - name: Check on failures
+        if: steps.bench.outcome != 'success'
+        run: |
+          cat /tmp/gc-referrers-bench-s3.log
+          exit 1
+
+  gc-stress-s3:
+    name: GC(without referrers) on S3(minio) with short interval
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/clean-runner
+      - uses: actions/setup-go@v4
+        with:
+          cache: false
+          go-version: 1.20.x
+      - name: Setup localstack service
+        run: |
+            pip install localstack                    # Install LocalStack cli
+            docker pull localstack/localstack:1.3     # Make sure to pull the latest version of the image
+            localstack start -d                       # Start LocalStack in the background
+            
+            echo "Waiting for LocalStack startup..."  # Wait 30 seconds for the LocalStack container
+            localstack wait -t 30                     # to become ready before timing out 
+            echo "Startup complete"
+            
+            # aws --endpoint-url=http://localhost:4566 s3api create-bucket --bucket zot-storage --region us-east-2 --create-bucket-configuration="{\"LocationConstraint\": \"us-east-2\"}"
+            aws dynamodb --endpoint-url http://localhost:4566 --region "us-east-2" create-table --table-name BlobTable --attribute-definitions AttributeName=Digest,AttributeType=S --key-schema AttributeName=Digest,KeyType=HASH --provisioned-throughput ReadCapacityUnits=10,WriteCapacityUnits=5
+        env:
+          AWS_ACCESS_KEY_ID: fake
+          AWS_SECRET_ACCESS_KEY: fake
+      - name: Setup minio service
+        run: |
+            docker run -d -p 9000:9000 --name minio \
+                -e "MINIO_ACCESS_KEY=minioadmin" \
+                -e "MINIO_SECRET_KEY=minioadmin" \
+                -v /tmp/data:/data \
+                -v /tmp/config:/root/.minio \
+                --health-cmd "curl http://localhost:9000/minio/health/live" \
+                minio/minio:edge-cicd server /data
+      - name: Install py minio
+        run: pip3 install minio
+
+      - name: Wait for minio to come up
+        run: |
+          sleep 10
+          curl --connect-timeout 5 \
+            --max-time 120 \
+            --retry 12 \
+            --retry-max-time 120 \
+            'http://localhost:9000/minio/health/live'
+
+      - name: Create minio bucket
+        run: |
+            python3 - <<'EOF'
+            from minio import Minio
+
+            try:
+                minio = Minio(
+                    'localhost:9000',
+                    access_key='minioadmin',
+                    secret_key='minioadmin',
+                    secure=False
+                )
+            except Exception as ex:
+                raise
+
+            minio.make_bucket('zot-storage')
+            print(f'{minio.list_buckets()}')
+            EOF
+
+      - name: Run zb
+        id: bench
+        run: |
+            make binary
+            make bench
+            ./bin/zot-linux-amd64 serve test/gc-stress/config-gc-bench-s3-minio.json &
+            sleep 10
+            bin/zb-linux-amd64 -c 10 -n 100 -o ci-cd http://localhost:8080
+
+            killall -r zot-*
+
+            # clean zot storage
+            sudo rm -rf /tmp/zot
+        env:
+          AWS_ACCESS_KEY_ID: fake
+          AWS_SECRET_ACCESS_KEY: fake
+        continue-on-error: true
+
+      - name: Check on failures
+        if: steps.bench.outcome != 'success'
+        run: |
+          cat /tmp/gc-bench-s3.log
+          exit 1
diff --git a/.github/workflows/nightly.yaml b/.github/workflows/nightly.yaml
@@ -72,7 +72,7 @@ jobs:
             make run-blackbox-sync-nightly
 
   gc-referrers-stress-s3:
-    name: GC(with referrers) on S3 with short interval
+    name: GC(with referrers) on S3(localstack) with short interval
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -96,11 +96,13 @@ jobs:
         env:
           AWS_ACCESS_KEY_ID: fake
           AWS_SECRET_ACCESS_KEY: fake
+
       - name: Run zb
+        id: bench
         run: |
             make binary
             make bench
-            ./bin/zot-linux-amd64 serve test/gc-stress/config-gc-referrers-bench-s3.json &
+            ./bin/zot-linux-amd64 serve test/gc-stress/config-gc-referrers-bench-s3-localstack.json &
             sleep 10
             bin/zb-linux-amd64 -c 10 -n 100 -o ci-cd http://localhost:8080
 
@@ -111,9 +113,16 @@ jobs:
         env:
           AWS_ACCESS_KEY_ID: fake
           AWS_SECRET_ACCESS_KEY: fake
+        continue-on-error: true
+
+      - name: Check on failures
+        if: steps.bench.outcome != 'success'
+        run: |
+          cat /tmp/gc-referrers-bench-s3.log
+          exit 1
 
   gc-stress-s3:
-    name: GC(without referrers) on S3 with short interval
+    name: GC(without referrers) on S3(localstack) with short interval
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -137,11 +146,13 @@ jobs:
         env:
           AWS_ACCESS_KEY_ID: fake
           AWS_SECRET_ACCESS_KEY: fake
+
       - name: Run zb
+        id: bench
         run: |
             make binary
             make bench
-            ./bin/zot-linux-amd64 serve test/gc-stress/config-gc-bench-s3.json &
+            ./bin/zot-linux-amd64 serve test/gc-stress/config-gc-bench-s3-localstack.json &
             sleep 10
             bin/zb-linux-amd64 -c 10 -n 100 -o ci-cd http://localhost:8080
 
@@ -152,3 +163,10 @@ jobs:
         env:
           AWS_ACCESS_KEY_ID: fake
           AWS_SECRET_ACCESS_KEY: fake
+        continue-on-error: true
+
+      - name: Check on failures
+        if: steps.bench.outcome != 'success'
+        run: |
+          cat /tmp/gc-bench-s3.log
+          exit 1
diff --git a/pkg/storage/imagestore/imagestore.go b/pkg/storage/imagestore/imagestore.go
@@ -672,7 +672,7 @@ func (is *ImageStore) deleteImageManifest(repo, reference string, detectCollisio
 	}
 
 	if _, err := is.storeDriver.WriteFile(file, buf); err != nil {
-		is.log.Debug().Str("deleting reference", reference).Msg("")
+		is.log.Debug().Str("reference", reference).Str("repo", repo).Msg("error while updating index.json")
 
 		return err
 	}
@@ -1660,7 +1660,6 @@ func (is *ImageStore) garbageCollectIndexReferrers(repo string, rootIndex ispec.
 			if gced {
 				count++
 			}
-
 		case ispec.MediaTypeImageManifest, artifactspec.MediaTypeArtifactManifest:
 			image, err := common.GetImageManifest(is, repo, desc.Digest, is.log)
 			if err != nil {
@@ -1890,8 +1889,13 @@ func (is *ImageStore) garbageCollectBlobs(imgStore *ImageStore, repo string,
 		}
 	}
 
+	blobUploads, err := is.storeDriver.List(path.Join(is.RootDir(), repo, storageConstants.BlobUploadDir))
+	if err != nil {
+		is.log.Debug().Str("repository", repo).Msg("unable to list .uploads/ dir")
+	}
+
 	// if we cleaned all blobs let's also remove the repo so that it won't be returned by catalog
-	if reaped == len(allBlobs) {
+	if len(allBlobs) > 0 && reaped == len(allBlobs) && len(blobUploads) == 0 {
 		log.Info().Str("repository", repo).Msg("garbage collected all blobs, cleaning repo...")
 
 		if err := is.storeDriver.Delete(path.Join(is.rootDir, repo)); err != nil {