update azure fragments

.github/workflows/ci-workflow.yaml

@@ -3,11 +3,14 @@ name: Gazette Continuous Integration
 # We build on any push to a branch, or when a release is created.
 on:
   pull_request:
+    branches:
+      - "arize"
     paths-ignore:
       - "docs/**"
   push:
     branches:
-      - "master"
+      - "arize"
+      - "arize-dev/*"
     # Ignore pushes to tags, since those ought to be handled by the release created event.
     tags-ignore:
       - "*"
@@ -17,13 +20,18 @@ on:
     # Without this additional restriction, GH actions will trigger multiple runs for a single
     # release, because it fires off separate events creating vs publishing the release.
     types: [created]
+  workflow_dispatch:
 
 env:
   # This is only used as the cache key to prevent rebuilding rocksdb every time. Eventually
   # we'll need to figure out a solution that doesn't duplicate this version everywhere.
   # For now, ensure that it's changed both here and in mk/common-config.mk.
   ROCKSDB_VERSION: "6.22.1"
 
+permissions:
+  id-token: write
+  contents: read
+
 jobs:
   build:
     name: "Build"
@@ -56,24 +64,33 @@ jobs:
             fi
           else
             # This is not a release, so we'll use 'dev-<sha>' for the version number
-            # and just 'latest-dev' for the docker tag.
+            # and just 'latest' for the docker tag.
             sha=${{ github.sha }}
             version="dev-${sha:0:7}"
             # If this is a master build, then we'll treat this as a release and just use the
             # hard-coded tag as the docker image tag.
             if [[ '${{ github.ref }}' == 'refs/heads/master' ]]; then
                 # We don't want to put the git sha in the docker tag because otherwise they'll
-                # accumulate forever and just clutter up the page on docker hub. So 'latest-dev'
+                # accumulate forever and just clutter up the page on docker hub. So 'latest'
                 # just always gets you the most recent master build, and if you want a specific master
                 # build, then you can use the '@sha256:...' syntax.
-                docker_tag="latest-dev"
+                docker_tag="${{ secrets.REGISTRY_PATH }}/gazette/broker:latest"
+                push_images='true'
+            elif [[ '${{ github.ref }}' == 'refs/heads/arize' ]]; then
+                version="0.89.1-arize-${sha:0:7}"
+                docker_tag="${{ secrets.REGISTRY_PATH }}/gazette/broker:arize-${sha:0:7}"
+                push_images='true'
+            elif [[ '${{ github.ref }}' == *'arize'* ]]; then
+                version="0.89.1-dev-${sha:0:7}"
+                docker_tag="${{ secrets.REGISTRY_PATH }}/gazette/broker:dev-${sha:0:7}"
                 push_images='true'
             else
+                docker_tag="latest"
                 push_images='false'
             fi
           fi
           echo ::set-output name=VERSION::${version}
-          echo ::set-output name=DOCKER_TAG::${docker_tag:-$version}
+          echo ::set-output name=DOCKER_TAG::${docker_tag}
           echo ::set-output name=PUSH_IMAGES::${push_images}
           echo ::set-output name=IS_RELEASE::${is_release}
 
@@ -105,6 +122,20 @@ jobs:
           # because go will use its own finer-grained cache invalidation logic.
           restore-keys: "go-mod-c4-"
 
+
+      - uses: 'google-github-actions/auth@v1'
+        with:
+          token_format: "access_token"          
+          project_id: ${{ secrets.PROJECT_ID }}
+          workload_identity_provider: projects/${{ secrets.PROJECT_NUMBER }}/locations/global/workloadIdentityPools/github/providers/github-actions
+          service_account: ${{ secrets.SERVICE_ACCOUNT }}
+
+      - name: 'Set up Cloud SDK'
+        uses: 'google-github-actions/setup-gcloud@v1'
+
+      - name: 'Use gcloud CLI'
+        run: gcloud info
+
       - name: "Build Binaries"
         run: "make as-ci target=release-linux-binaries VERSION=${{ steps.release_info.outputs.VERSION }}"
 
@@ -130,10 +161,11 @@ jobs:
           upload_url: "${{ github.event.release.upload_url }}"
           asset_content_type: application/zip
 
+
       - name: "Build and Push Docker Images"
         if: steps.release_info.outputs.PUSH_IMAGES == 'true'
         run: |
-          docker login -u '${{ secrets.DOCKER_USERNAME }}' -p '${{ secrets.DOCKER_PASSWORD }}' ${{ secrets.DOCKER_REGISTRY }}
-          make as-ci target=ci-release-gazette-examples VERSION=${{ steps.release_info.outputs.VERSION }}
           make as-ci target=ci-release-gazette-broker VERSION=${{ steps.release_info.outputs.VERSION }}
-          make push-to-registry REGISTRY=${{ secrets.DOCKER_REGISTRY }} RELEASE_TAG=${{ steps.release_info.outputs.DOCKER_TAG }}
+          docker tag gazette/broker:latest ${{ steps.release_info.outputs.DOCKER_TAG }}
+          gcloud auth configure-docker ${{ secrets.REGISTRY }}
+          docker push ${{ steps.release_info.outputs.DOCKER_TAG }}

broker/fragment/store_azure.go

@@ -48,6 +48,8 @@ type azureBackend struct {
 	clientMu  sync.Mutex
 	udc       *service.UserDelegationCredential
 	udcExp    *time.Time
+
+	sharedKeyCredentials *sas.SharedKeyCredential
 }
 
 func (a *azureBackend) Provider() string {
@@ -57,31 +59,56 @@ func (a *azureBackend) Provider() string {
 // See here for an example of how to use the Azure client libraries to create signatures:
 // https://github.com/Azure/azure-sdk-for-go/blob/main/sdk/storage/azblob/service/examples_test.go#L285
 func (a *azureBackend) SignGet(ep *url.URL, fragment pb.Fragment, d time.Duration) (string, error) {
+	var (
+		sasQueryParams sas.QueryParameters
+		err            error
+	)
+
 	cfg, _, err := a.azureClient(ep)
 	if err != nil {
 		return "", err
 	}
 	blobName := cfg.rewritePath(cfg.prefix, fragment.ContentPath())
 
-	udc, err := a.getUserDelegationCredential()
-	if err != nil {
-		return "", err
-	}
+	if ep.Scheme == "azure" {
+		// Note: for arize we assume azure scheme is for blob SAS (as opposed to container SAS in azure-ad case)
+		perms := sas.BlobPermissions{Add: true, Read: true, Write: true}
 
-	sasQueryParams, err := sas.BlobSignatureValues{
-		Protocol:      sas.ProtocolHTTPS,       // Users MUST use HTTPS (not HTTP)
-		ExpiryTime:    time.Now().UTC().Add(d), // Timestamps are expected in UTC https://docs.microsoft.com/en-us/rest/api/storageservices/create-service-sas#service-sas-example
-		ContainerName: cfg.containerName,
-		BlobName:      blobName,
+		sasQueryParams, err = sas.BlobSignatureValues{
+			Protocol:      sas.ProtocolHTTPS, // Users MUST use HTTPS (not HTTP)
+			ExpiryTime:    time.Now().UTC().Add(d),
+			ContainerName: cfg.containerName,
+			BlobName:      blobName,
+			Permissions:   perms.String(),
+		}.SignWithSharedKey(a.sharedKeyCredentials)
 
-		// To produce a container SAS (as opposed to a blob SAS), assign to Permissions using
-		// ContainerSASPermissions and make sure the BlobName field is "" (the default).
-		Permissions: to.Ptr(sas.ContainerPermissions{Read: true, Add: true, Write: true}).String(),
-	}.SignWithUserDelegation(udc)
+		if err != nil {
+			return "", err
+		}
+	} else if ep.Scheme == "azure-ad" {
+		udc, err := a.getUserDelegationCredential()
+		if err != nil {
+			return "", err
+		}
 
-	if err != nil {
-		return "", err
+		sasQueryParams, err = sas.BlobSignatureValues{
+			Protocol:      sas.ProtocolHTTPS,       // Users MUST use HTTPS (not HTTP)
+			ExpiryTime:    time.Now().UTC().Add(d), // Timestamps are expected in UTC https://docs.microsoft.com/en-us/rest/api/storageservices/create-service-sas#service-sas-example
+			ContainerName: cfg.containerName,
+			BlobName:      blobName,
+
+			// To produce a container SAS (as opposed to a blob SAS), assign to Permissions using
+			// ContainerSASPermissions and make sure the BlobName field is "" (the default).
+			Permissions: to.Ptr(sas.ContainerPermissions{Read: true, Add: true, Write: true}).String(),
+		}.SignWithUserDelegation(udc)
+
+		if err != nil {
+			return "", err
+		}
+	} else {
+		return "", fmt.Errorf("unknown scheme: %s", ep.Scheme)
 	}
+
 	return fmt.Sprintf("%s/%s?%s", cfg.containerURL(), blobName, sasQueryParams.Encode()), nil
 }
 
@@ -258,6 +285,7 @@ func (a *azureBackend) azureClient(ep *url.URL) (cfg AzureStoreConfig, client pi
 		if err != nil {
 			return cfg, nil, err
 		}
+		a.sharedKeyCredentials = sharedKeyCred // Arize addition
 		serviceClient, err := service.NewClientWithSharedKeyCredential(cfg.serviceUrl(), sharedKeyCred, &service.ClientOptions{})
 		if err != nil {
 			return cfg, nil, err

broker/fragment/store_s3.go

@@ -220,7 +220,6 @@ func (s *s3Backend) s3Client(ep *url.URL) (cfg S3StoreConfig, client *s3.S3, err
 	}
 
 	awsSession, err := session.NewSessionWithOptions(session.Options{
-		Config:  *awsConfig,
 		Profile: cfg.Profile,
 	})
 	if err != nil {
@@ -250,7 +249,7 @@ func (s *s3Backend) s3Client(ep *url.URL) (cfg S3StoreConfig, client *s3.S3, err
 		"providerName": creds.ProviderName,
 	}).Info("constructed new aws.Session")
 
-	client = s3.New(awsSession)
+	client = s3.New(awsSession, awsConfig)
 	s.clients[key] = client
 
 	return

broker/protocol/dispatcher.go

@@ -117,12 +117,19 @@ func (d *dispatcher) UpdateSubConnState(sc balancer.SubConn, state balancer.SubC
 		panic("unexpected SubConn")
 	}
 
-	if state.ConnectivityState == connectivity.Connecting && d.connState[sc] == connectivity.TransientFailure {
+	if (state.ConnectivityState == connectivity.Connecting || state.ConnectivityState == connectivity.Idle) &&
+		d.connState[sc] == connectivity.TransientFailure {
 		// gRPC will quickly transition failed connections back into a Connecting
 		// state. In many cases, such as a remote-initiated close from a
 		// shutting-down server, the SubConn may never return. Until we see a
 		// successful re-connect, continue to consider the SubConn as broken
 		// (and trigger invalidations of cached Routes which use it).
+
+		if state.ConnectivityState == connectivity.Idle {
+			sc.Connect()
+		}
+		d.mu.Unlock()
+		return
 	} else {
 		d.connState[sc] = state.ConnectivityState
 	}
@@ -132,6 +139,10 @@ func (d *dispatcher) UpdateSubConnState(sc balancer.SubConn, state balancer.SubC
 		delete(d.connID, sc)
 		delete(d.connState, sc)
 	}
+
+	if state.ConnectivityState == connectivity.Idle {
+		sc.Connect()
+	}
 	d.mu.Unlock()
 
 	// Notify gRPC that block requests may now be able to proceed.

go.mod

@@ -29,7 +29,7 @@ require (
 	github.com/mattn/go-sqlite3 v2.0.3+incompatible
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/pkg/errors v0.9.1
-	github.com/prometheus/client_golang v1.11.0
+	github.com/prometheus/client_golang v1.11.1
 	github.com/prometheus/client_model v0.2.0
 	github.com/sirupsen/logrus v1.8.1
 	github.com/soheilhy/cmux v0.1.5

go.sum

@@ -342,8 +342,9 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
 github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
-github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ=
 github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
+github.com/prometheus/client_golang v1.11.1 h1:+4eQaD7vAZ6DsfsxB15hbE0odUjGI5ARs9yskGu1v4s=
+github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=

mk/ci-release.Dockerfile

@@ -18,6 +18,10 @@ RUN apt-get update -y \
 COPY * /usr/local/bin/
 RUN mv /usr/local/bin/librocksdb.so* /usr/local/lib/ && ldconfig
 
+# Arize - remove a few base utilities flagged by security scans as having suid/sgid set.
+# Note: we did not see these bits set ourselves when deploying in our test cluster.
+RUN rm -f /usr/bin/mount /usr/bin/umount /usr/bin/su /usr/bin/wall
+
 # Run as non-privileged "gazette" user.
 RUN useradd gazette --create-home --shell /usr/sbin/nologin
 USER gazette

mk/common-build.mk

@@ -80,6 +80,7 @@ ci-release-%: $(ROCKSDIR)/librocksdb.so go-install $$($$@-targets)
 	ln ${$@-targets} ${ROCKSDIR}/librocksdb.so.${ROCKSDB_VERSION} \
 		${WORKDIR}/ci-release
 	docker build \
+		--no-cache \
 		-f ${COREDIR}/mk/ci-release.Dockerfile \
 		-t $(subst -,/,$*):latest \
 		${WORKDIR}/ci-release/