WIP: Azure Blob Storage implementation

.github/actions/setup-localstack/action.yaml

@@ -36,3 +36,19 @@ runs:
       env:
         AWS_ACCESS_KEY_ID: fake
         AWS_SECRET_ACCESS_KEY: fake
+
+    - shell: bash
+      run: |
+        docker run -d -p 10000:10000 -p 10001:10001 -p 10002:10002 --name azurite \
+            -v /tmp/data:/data \
+            -e "AZURITE_ACCOUNTS=zot-storage-test:YXp1cml0ZQo=" \
+            mcr.microsoft.com/azure-storage/azurite:3.31.0 azurite-blob --blobHost 0.0.0.0
+
+
+    - shell: bash
+      run: |
+        sleep 10
+        az storage container create --name zot-storage
+      env:
+        # https://github.com/Azure/Azurite/blob/main/README.md#usage-with-azure-storage-sdks-or-tools
+        AZURE_STORAGE_CONNECTION_STRING: DefaultEndpointsProtocol=http;AccountName=zot-storage-test;AccountKey=YXp1cml0ZQo=;BlobEndpoint=http://127.0.0.1:10000/zot-storage-test;

.github/actions/teardown-localstack/action.yaml

@@ -6,3 +6,7 @@ runs:
     - shell: bash
       if: always()
       run: localstack stop
+
+    - shell: bash
+      if: always()
+      run: docker rm -f azurite

.github/workflows/gc-stress-test.yaml

@@ -109,7 +109,7 @@ jobs:
         run: |
             python3 - <<'EOF'
             from minio import Minio
-
+            
             try:
                 minio = Minio(
                     'localhost:9000',
@@ -119,7 +119,7 @@ jobs:
                 )
             except Exception as ex:
                 raise
-
+            
             minio.make_bucket('zot-storage')
             print(f'{minio.list_buckets()}')
             EOF
@@ -132,9 +132,9 @@ jobs:
             ./bin/zot-linux-amd64 serve test/gc-stress/config-gc-referrers-bench-s3-minio.json &
             sleep 10
             bin/zb-linux-amd64 -c 10 -n 100 -o ci-cd http://localhost:8080
-
+            
             killall -r zot-*
-
+            
             # clean zot storage
             sudo rm -rf /tmp/zot
         env:
@@ -184,7 +184,7 @@ jobs:
         run: |
             python3 - <<'EOF'
             from minio import Minio
-
+            
             try:
                 minio = Minio(
                     'localhost:9000',
@@ -194,7 +194,7 @@ jobs:
                 )
             except Exception as ex:
                 raise
-
+            
             minio.make_bucket('zot-storage')
             print(f'{minio.list_buckets()}')
             EOF
@@ -207,9 +207,9 @@ jobs:
             ./bin/zot-linux-amd64 serve test/gc-stress/config-gc-bench-s3-minio.json &
             sleep 10
             bin/zb-linux-amd64 -c 10 -n 100 -o ci-cd http://localhost:8080
-
+            
             killall -r zot-*
-
+            
             # clean zot storage
             sudo rm -rf /tmp/zot
         env:
@@ -223,3 +223,94 @@ jobs:
           cat /tmp/gc-bench-s3.log
           exit 1
       - uses: ./.github/actions/teardown-localstack
+  gc-referrers-stress-azure-blob:
+    name: GC(with referrers) on azure-blob(azurite) with short interval
+    runs-on: ubuntu-latest-8-cores
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          cache: false
+          go-version: 1.22.x
+      - name: Setup azurite service
+        run: |
+            docker run -d -p 10000:10000 -p 10001:10001 -p 10002:10002 --name azurite \
+                -v /tmp/data:/data \
+                -e "AZURITE_ACCOUNTS=zot-storage-test:YXp1cml0ZQo=" \
+                mcr.microsoft.com/azure-storage/azurite:3.31.0 azurite-blob --blobHost 0.0.0.0
+
+      - name: Wait for azurite to come up
+        run: |
+            sleep 10
+            az storage container create --name zot-storage-test
+        env:
+          # https://github.com/Azure/Azurite/blob/main/README.md#usage-with-azure-storage-sdks-or-tools
+          AZURE_STORAGE_CONNECTION_STRING: DefaultEndpointsProtocol=http;AccountName=zot-storage-test;AccountKey=YXp1cml0ZQo=;BlobEndpoint=http://localhost:10000/zot-storage-test;
+
+      - name: Run zb
+        id: bench
+        run: |
+            make binary
+            make bench
+            ./bin/zot-linux-amd64 serve test/gc-stress/config-gc-referrers-bench-azure-blob-azurite.json &
+            sleep 10
+            bin/zb-linux-amd64 -c 10 -n 100 -o ci-cd http://localhost:8080
+
+            killall -r zot-*
+
+            # clean zot storage
+            sudo rm -rf /tmp/zot
+        env:
+          AWS_ACCESS_KEY_ID: fake
+          AWS_SECRET_ACCESS_KEY: fake
+        continue-on-error: true
+
+      - name: Check on failures
+        if: steps.bench.outcome != 'success'
+        run: |
+          cat /tmp/gc-referrers-bench-azure-blob.log
+          exit 1
+  gc-stress-azure-blob:
+    name: GC(without referrers) on azure-blob(azurite) with short interval
+    runs-on: ubuntu-latest-8-cores
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          cache: false
+          go-version: 1.22.x
+      - name: Setup azurite service
+        run: |
+            docker run -d -p 10000:10000 -p 10001:10001 -p 10002:10002 --name azurite \
+                -v /tmp/data:/data \
+                -e "AZURITE_ACCOUNTS=zot-storage-test:YXp1cml0ZQo=" \
+                mcr.microsoft.com/azure-storage/azurite:3.31.0 azurite-blob --blobHost 0.0.0.0
+
+      - name: Wait for azurite to come up
+        run: |
+            sleep 10
+            az storage container create --name zot-storage-test
+        env:
+          # https://github.com/Azure/Azurite/blob/main/README.md#usage-with-azure-storage-sdks-or-tools
+          AZURE_STORAGE_CONNECTION_STRING: DefaultEndpointsProtocol=http;AccountName=zot-storage-test;AccountKey=YXp1cml0ZQo=;BlobEndpoint=http://127.0.0.1:10000/zot-storage-test;
+
+      - name: Run zb
+        id: bench
+        run: |
+            make binary
+            make bench
+            ./bin/zot-linux-amd64 serve test/gc-stress/config-gc-referrers-bench-azure-blob-azurite.json &
+            sleep 10
+            bin/zb-linux-amd64 -c 10 -n 100 -o ci-cd http://localhost:8080
+
+            killall -r zot-*
+
+            # clean zot storage
+            sudo rm -rf /tmp/zot
+        continue-on-error: true
+
+      - name: Check on failures
+        if: steps.bench.outcome != 'success'
+        run: |
+          cat /tmp/gc-bench-azure-blob.log
+          exit 1

.github/workflows/test.yaml

@@ -44,6 +44,7 @@ jobs:
         env:
           S3MOCK_ENDPOINT: localhost:4566
           DYNAMODBMOCK_ENDPOINT: http://localhost:4566
+          AZURE_BLOB_MOCK_ENDPOINT: http://localhost:10000/
           AWS_ACCESS_KEY_ID: fake
           AWS_SECRET_ACCESS_KEY: fake
       - name: upload coverage
@@ -83,6 +84,7 @@ jobs:
         env:
           S3MOCK_ENDPOINT: localhost:4566
           DYNAMODBMOCK_ENDPOINT: http://localhost:4566
+          AZURE_BLOB_MOCK_ENDPOINT: http://localhost:10000/
           AWS_ACCESS_KEY_ID: fake
           AWS_SECRET_ACCESS_KEY: fake
       - name: upload coverage

examples/config-azure.json

@@ -0,0 +1,64 @@
+{
+    "distSpecVersion": "1.1.0",
+    "storage": {
+        "rootDirectory": "/tmp/zot",
+        "dedupe": false,
+        "remoteCache": false,
+        "storageDriver": {
+            "name": "azure",
+            "rootdirectory": "/zot",
+            "container": "zot-storage",
+            "accountname": "zot-storage",
+            "accountkey": "azurite",
+            "serviceurl": "http://localhost:10000"
+        },
+        "subPaths": {
+            "/a": {
+                "rootDirectory": "/tmp/zot1",
+                "dedupe": false,
+                "remoteCache": false,
+                "storageDriver": {
+                    "name": "azure",
+                    "rootdirectory": "/zot-a",
+                    "container": "zot-storage",
+                    "accountname": "zot-storage",
+                    "accountkey": "azurite",
+                    "serviceurl": "http://localhost:10000"
+                }
+            },
+            "/b": {
+                "rootDirectory": "/tmp/zot2",
+                "dedupe": false,
+                "remoteCache": false,
+                "storageDriver": {
+                    "name": "azure",
+                    "rootdirectory": "/zot-b",
+                    "container": "zot-storage",
+                    "accountname": "zot-storage",
+                    "accountkey": "azurite",
+                    "serviceurl": "http://localhost:10000"
+                }
+            },
+            "/c": {
+                "rootDirectory": "/tmp/zot3",
+                "dedupe": false,
+                "remoteCache": false,
+                "storageDriver": {
+                    "name": "azure",
+                    "rootdirectory": "/zot-c",
+                    "container": "zot-storage",
+                    "accountname": "zot-storage",
+                    "accountkey": "azurite",
+                    "serviceurl": "http://localhost:10000"
+                }
+            }
+        }
+    },
+    "http": {
+        "address": "127.0.0.1",
+        "port": "8080"
+    },
+    "log": {
+        "level": "debug"
+    }
+}

go.mod

@@ -88,6 +88,7 @@ require (
 	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.13.0 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.0 // indirect
 	github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect
 	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
 	github.com/Azure/go-autorest/autorest v0.11.29 // indirect

go.sum

@@ -217,10 +217,14 @@ github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 h1:tfLQ34V6F7tVSwoTf/4lH
 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0/go.mod h1:9kIvujWAA58nmPmWB1m23fyWic1kYZMxD9CxaWn4Qpg=
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xPBn1663uRv2t2q/ESv9seY=
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0/go.mod h1:iZDifYGJTIgIIkYRNWPENUnqx6bJ2xnSDFI2tjwZNuY=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0 h1:PiSrjRPpkQNjrM8H0WwKMnZUdu1RGMtd/LdGKUrOo+c=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0/go.mod h1:oDrbWx4ewMylP7xHivfgixbfGBT6APAwsSoHRKotnIc=
 github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.1.0 h1:DRiANoJTiW6obBQe3SqZizkuV1PEgfiiGivmVocDy64=
 github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.1.0/go.mod h1:qLIye2hwb/ZouqhpSD9Zn3SJipvpEnz1Ywl3VUk9Y0s=
 github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.0.0 h1:D3occbWoio4EBLkbkevetNMAVX197GkzbUMtqjGWn80=
 github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.0.0/go.mod h1:bTSOgj05NGRuHHhQwAdPnYr9TOdNmKlZTgGLL6nyAdI=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.0 h1:Be6KInmFEKV81c0pOAEbRYehLMwmmGI1exuFj248AMk=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.0/go.mod h1:WCPBHsOXfBVnivScjs2ypRfimjEW0qPVLGgJkZlrIOA=
 github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0=
 github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
 github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs=

pkg/cli/server/extensions_test.go

@@ -40,6 +40,7 @@ func TestVerifyExtensionsConfig(t *testing.T) {
 					"rootdirectory":"/zot",
 					"region":"us-east-2",
 					"bucket":"zot-storage",
+					"forcepathstyle":true,
 					"secure":true,
 					"skipverify":false
 				}
@@ -79,6 +80,7 @@ func TestVerifyExtensionsConfig(t *testing.T) {
 							"rootdirectory":"/zot-a",
 							"region":"us-east-2",
 							"bucket":"zot-storage",
+							"forcepathstyle":true,
 							"secure":true,
 							"skipverify":false
 						}
@@ -1897,6 +1899,7 @@ func TestSyncWithRemoteStorageConfig(t *testing.T) {
 							"rootdirectory":"/zot-a",
 							"region":"us-east-2",
 							"bucket":"zot-storage",
+							"forcepathstyle":true,
 							"secure":true,
 							"skipverify":true
 						}

pkg/cli/server/root.go

@@ -401,8 +401,10 @@ func validateConfiguration(config *config.Config, log zlog.Logger) error {
 	}
 
 	if len(config.Storage.StorageDriver) != 0 {
-		// enforce s3 driver in case of using storage driver
-		if config.Storage.StorageDriver["name"] != storageConstants.S3StorageDriverName {
+		switch config.Storage.StorageDriver["name"] {
+		case storageConstants.S3StorageDriverName:
+		case storageConstants.AzureBlobStorageDriverName:
+		default:
 			log.Error().Err(zerr.ErrBadConfig).Interface("cacheDriver", config.Storage.StorageDriver["name"]).
 				Msg("unsupported storage driver")
 
@@ -425,14 +427,17 @@ func validateConfiguration(config *config.Config, log zlog.Logger) error {
 
 			for route, storageConfig := range subPaths {
 				if len(storageConfig.StorageDriver) != 0 {
-					if storageConfig.StorageDriver["name"] != storageConstants.S3StorageDriverName {
+					switch storageConfig.StorageDriver["name"] {
+					case storageConstants.S3StorageDriverName:
+					case storageConstants.AzureBlobStorageDriverName:
+					default:
 						log.Error().Err(zerr.ErrBadConfig).Str("subpath", route).Interface("storageDriver",
 							storageConfig.StorageDriver["name"]).Msg("unsupported storage driver")
 
 						return zerr.ErrBadConfig
 					}
 
-					// enforce tmpDir in case sync + s3
+					// enforce tmpDir in case sync + blob storage
 					if config.Extensions != nil && config.Extensions.Sync != nil && config.Extensions.Sync.DownloadDir == "" {
 						log.Error().Err(zerr.ErrBadConfig).
 							Msg("using both sync and remote storage features needs config.Extensions.Sync.DownloadDir to be specified")

pkg/cli/server/root_test.go

@@ -197,6 +197,7 @@ storage:
 					"rootdirectory":"/zot",
 					"region":"us-east-2",
 					"bucket":"zot-storage",
+					"forcepathstyle":true,
 					"secure":true,
 					"skipverify":false
 				}
@@ -235,6 +236,7 @@ storage:
 							"rootdirectory":"/zot-a",
 							"region":"us-east-2",
 							"bucket":"zot-storage",
+							"forcepathstyle":true,
 							"secure":true,
 							"skipverify":false
 						}
@@ -278,6 +280,7 @@ storage:
 					"rootdirectory":"/zot",
 					"region":"us-east-2",
 					"bucket":"zot-storage",
+					"forcepathstyle":true,
 					"secure":true,
 					"skipverify":false
 				}
@@ -347,6 +350,7 @@ storage:
 				  "rootdirectory":"/zot",
 				  "region":"us-east-2",
 				  "bucket":"zot-storage",
+				  "forcepathstyle":true,
 				  "secure":true,
 				  "skipverify":false
 			   }
@@ -387,6 +391,7 @@ storage:
 				  "rootdirectory":"/zot",
 				  "region":"us-east-2",
 				  "bucket":"zot-storage",
+				  "forcepathstyle":true,
 				  "secure":true,
 				  "skipverify":false
 			   }
@@ -427,6 +432,7 @@ storage:
 						"rootdirectory":"/zot",
 						"region":"us-east-2",
 						"bucket":"zot-storage",
+						"forcepathstyle":true,
 						"secure":true,
 						"skipverify":false
 					 }
@@ -508,6 +514,7 @@ storage:
 						"rootdirectory":"/zot",
 						"region":"us-east-2",
 						"bucket":"zot-storage",
+						"forcepathstyle":true,
 						"secure":true,
 						"skipverify":false
 					 }