diff --git a/.github/workflows/ci-workflow.yaml b/.github/workflows/ci-workflow.yaml index 3442df82..0ee1ac26 100644 --- a/.github/workflows/ci-workflow.yaml +++ b/.github/workflows/ci-workflow.yaml @@ -3,11 +3,14 @@ name: Gazette Continuous Integration # We build on any push to a branch, or when a release is created. on: pull_request: + branches: + - "arize" paths-ignore: - "docs/**" push: branches: - - "master" + - "arize" + - "arize-dev/*" # Ignore pushes to tags, since those ought to be handled by the release created event. tags-ignore: - "*" @@ -17,6 +20,7 @@ on: # Without this additional restriction, GH actions will trigger multiple runs for a single # release, because it fires off separate events creating vs publishing the release. types: [created] + workflow_dispatch: env: # This is only used as the cache key to prevent rebuilding rocksdb every time. Eventually @@ -24,6 +28,10 @@ env: # For now, ensure that it's changed both here and in mk/common-config.mk. ROCKSDB_VERSION: "6.22.1" +permissions: + id-token: write + contents: read + jobs: build: name: "Build" @@ -56,24 +64,33 @@ jobs: fi else # This is not a release, so we'll use 'dev-' for the version number - # and just 'latest-dev' for the docker tag. + # and just 'latest' for the docker tag. sha=${{ github.sha }} version="dev-${sha:0:7}" # If this is a master build, then we'll treat this as a release and just use the # hard-coded tag as the docker image tag. if [[ '${{ github.ref }}' == 'refs/heads/master' ]]; then # We don't want to put the git sha in the docker tag because otherwise they'll - # accumulate forever and just clutter up the page on docker hub. So 'latest-dev' + # accumulate forever and just clutter up the page on docker hub. So 'latest' # just always gets you the most recent master build, and if you want a specific master # build, then you can use the '@sha256:...' syntax. - docker_tag="latest-dev" + docker_tag="${{ secrets.REGISTRY_PATH }}/gazette/broker:latest" + push_images='true' + elif [[ '${{ github.ref }}' == 'refs/heads/arize' ]]; then + version="0.89.1-arize-${sha:0:7}" + docker_tag="${{ secrets.REGISTRY_PATH }}/gazette/broker:arize-${sha:0:7}" + push_images='true' + elif [[ '${{ github.ref }}' == *'arize'* ]]; then + version="0.89.1-dev-${sha:0:7}" + docker_tag="${{ secrets.REGISTRY_PATH }}/gazette/broker:dev-${sha:0:7}" push_images='true' else + docker_tag="latest" push_images='false' fi fi echo ::set-output name=VERSION::${version} - echo ::set-output name=DOCKER_TAG::${docker_tag:-$version} + echo ::set-output name=DOCKER_TAG::${docker_tag} echo ::set-output name=PUSH_IMAGES::${push_images} echo ::set-output name=IS_RELEASE::${is_release} @@ -105,6 +122,20 @@ jobs: # because go will use its own finer-grained cache invalidation logic. restore-keys: "go-mod-c4-" + + - uses: 'google-github-actions/auth@v1' + with: + token_format: "access_token" + project_id: ${{ secrets.PROJECT_ID }} + workload_identity_provider: projects/${{ secrets.PROJECT_NUMBER }}/locations/global/workloadIdentityPools/github/providers/github-actions + service_account: ${{ secrets.SERVICE_ACCOUNT }} + + - name: 'Set up Cloud SDK' + uses: 'google-github-actions/setup-gcloud@v1' + + - name: 'Use gcloud CLI' + run: gcloud info + - name: "Build Binaries" run: "make as-ci target=release-linux-binaries VERSION=${{ steps.release_info.outputs.VERSION }}" @@ -130,10 +161,11 @@ jobs: upload_url: "${{ github.event.release.upload_url }}" asset_content_type: application/zip + - name: "Build and Push Docker Images" if: steps.release_info.outputs.PUSH_IMAGES == 'true' run: | - docker login -u '${{ secrets.DOCKER_USERNAME }}' -p '${{ secrets.DOCKER_PASSWORD }}' ${{ secrets.DOCKER_REGISTRY }} - make as-ci target=ci-release-gazette-examples VERSION=${{ steps.release_info.outputs.VERSION }} make as-ci target=ci-release-gazette-broker VERSION=${{ steps.release_info.outputs.VERSION }} - make push-to-registry REGISTRY=${{ secrets.DOCKER_REGISTRY }} RELEASE_TAG=${{ steps.release_info.outputs.DOCKER_TAG }} + docker tag gazette/broker:latest ${{ steps.release_info.outputs.DOCKER_TAG }} + gcloud auth configure-docker ${{ secrets.REGISTRY }} + docker push ${{ steps.release_info.outputs.DOCKER_TAG }} diff --git a/broker/fragment/store_azure.go b/broker/fragment/store_azure.go index cc55005f..e97ebd9d 100644 --- a/broker/fragment/store_azure.go +++ b/broker/fragment/store_azure.go @@ -48,6 +48,8 @@ type azureBackend struct { clientMu sync.Mutex udc *service.UserDelegationCredential udcExp *time.Time + + sharedKeyCredentials *sas.SharedKeyCredential } func (a *azureBackend) Provider() string { @@ -57,31 +59,56 @@ func (a *azureBackend) Provider() string { // See here for an example of how to use the Azure client libraries to create signatures: // https://github.com/Azure/azure-sdk-for-go/blob/main/sdk/storage/azblob/service/examples_test.go#L285 func (a *azureBackend) SignGet(ep *url.URL, fragment pb.Fragment, d time.Duration) (string, error) { + var ( + sasQueryParams sas.QueryParameters + err error + ) + cfg, _, err := a.azureClient(ep) if err != nil { return "", err } blobName := cfg.rewritePath(cfg.prefix, fragment.ContentPath()) - udc, err := a.getUserDelegationCredential() - if err != nil { - return "", err - } + if ep.Scheme == "azure" { + // Note: for arize we assume azure scheme is for blob SAS (as opposed to container SAS in azure-ad case) + perms := sas.BlobPermissions{Add: true, Read: true, Write: true} - sasQueryParams, err := sas.BlobSignatureValues{ - Protocol: sas.ProtocolHTTPS, // Users MUST use HTTPS (not HTTP) - ExpiryTime: time.Now().UTC().Add(d), // Timestamps are expected in UTC https://docs.microsoft.com/en-us/rest/api/storageservices/create-service-sas#service-sas-example - ContainerName: cfg.containerName, - BlobName: blobName, + sasQueryParams, err = sas.BlobSignatureValues{ + Protocol: sas.ProtocolHTTPS, // Users MUST use HTTPS (not HTTP) + ExpiryTime: time.Now().UTC().Add(d), + ContainerName: cfg.containerName, + BlobName: blobName, + Permissions: perms.String(), + }.SignWithSharedKey(a.sharedKeyCredentials) - // To produce a container SAS (as opposed to a blob SAS), assign to Permissions using - // ContainerSASPermissions and make sure the BlobName field is "" (the default). - Permissions: to.Ptr(sas.ContainerPermissions{Read: true, Add: true, Write: true}).String(), - }.SignWithUserDelegation(udc) + if err != nil { + return "", err + } + } else if ep.Scheme == "azure-ad" { + udc, err := a.getUserDelegationCredential() + if err != nil { + return "", err + } - if err != nil { - return "", err + sasQueryParams, err = sas.BlobSignatureValues{ + Protocol: sas.ProtocolHTTPS, // Users MUST use HTTPS (not HTTP) + ExpiryTime: time.Now().UTC().Add(d), // Timestamps are expected in UTC https://docs.microsoft.com/en-us/rest/api/storageservices/create-service-sas#service-sas-example + ContainerName: cfg.containerName, + BlobName: blobName, + + // To produce a container SAS (as opposed to a blob SAS), assign to Permissions using + // ContainerSASPermissions and make sure the BlobName field is "" (the default). + Permissions: to.Ptr(sas.ContainerPermissions{Read: true, Add: true, Write: true}).String(), + }.SignWithUserDelegation(udc) + + if err != nil { + return "", err + } + } else { + return "", fmt.Errorf("unknown scheme: %s", ep.Scheme) } + return fmt.Sprintf("%s/%s?%s", cfg.containerURL(), blobName, sasQueryParams.Encode()), nil } @@ -258,6 +285,7 @@ func (a *azureBackend) azureClient(ep *url.URL) (cfg AzureStoreConfig, client pi if err != nil { return cfg, nil, err } + a.sharedKeyCredentials = sharedKeyCred // Arize addition serviceClient, err := service.NewClientWithSharedKeyCredential(cfg.serviceUrl(), sharedKeyCred, &service.ClientOptions{}) if err != nil { return cfg, nil, err diff --git a/broker/fragment/store_s3.go b/broker/fragment/store_s3.go index 1a700f03..4488625f 100644 --- a/broker/fragment/store_s3.go +++ b/broker/fragment/store_s3.go @@ -220,7 +220,6 @@ func (s *s3Backend) s3Client(ep *url.URL) (cfg S3StoreConfig, client *s3.S3, err } awsSession, err := session.NewSessionWithOptions(session.Options{ - Config: *awsConfig, Profile: cfg.Profile, }) if err != nil { @@ -250,7 +249,7 @@ func (s *s3Backend) s3Client(ep *url.URL) (cfg S3StoreConfig, client *s3.S3, err "providerName": creds.ProviderName, }).Info("constructed new aws.Session") - client = s3.New(awsSession) + client = s3.New(awsSession, awsConfig) s.clients[key] = client return diff --git a/broker/protocol/dispatcher.go b/broker/protocol/dispatcher.go index e6fb35ae..46d73e92 100644 --- a/broker/protocol/dispatcher.go +++ b/broker/protocol/dispatcher.go @@ -117,12 +117,19 @@ func (d *dispatcher) UpdateSubConnState(sc balancer.SubConn, state balancer.SubC panic("unexpected SubConn") } - if state.ConnectivityState == connectivity.Connecting && d.connState[sc] == connectivity.TransientFailure { + if (state.ConnectivityState == connectivity.Connecting || state.ConnectivityState == connectivity.Idle) && + d.connState[sc] == connectivity.TransientFailure { // gRPC will quickly transition failed connections back into a Connecting // state. In many cases, such as a remote-initiated close from a // shutting-down server, the SubConn may never return. Until we see a // successful re-connect, continue to consider the SubConn as broken // (and trigger invalidations of cached Routes which use it). + + if state.ConnectivityState == connectivity.Idle { + sc.Connect() + } + d.mu.Unlock() + return } else { d.connState[sc] = state.ConnectivityState } @@ -132,6 +139,10 @@ func (d *dispatcher) UpdateSubConnState(sc balancer.SubConn, state balancer.SubC delete(d.connID, sc) delete(d.connState, sc) } + + if state.ConnectivityState == connectivity.Idle { + sc.Connect() + } d.mu.Unlock() // Notify gRPC that block requests may now be able to proceed. diff --git a/go.mod b/go.mod index 40ff71fe..00b9bf49 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,7 @@ require ( github.com/mattn/go-sqlite3 v2.0.3+incompatible github.com/olekukonko/tablewriter v0.0.5 github.com/pkg/errors v0.9.1 - github.com/prometheus/client_golang v1.11.0 + github.com/prometheus/client_golang v1.11.1 github.com/prometheus/client_model v0.2.0 github.com/sirupsen/logrus v1.8.1 github.com/soheilhy/cmux v0.1.5 diff --git a/go.sum b/go.sum index 52383680..3fdeaad1 100644 --- a/go.sum +++ b/go.sum @@ -342,8 +342,9 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_golang v1.11.1 h1:+4eQaD7vAZ6DsfsxB15hbE0odUjGI5ARs9yskGu1v4s= +github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= diff --git a/mk/ci-release.Dockerfile b/mk/ci-release.Dockerfile index 02365593..3853d155 100644 --- a/mk/ci-release.Dockerfile +++ b/mk/ci-release.Dockerfile @@ -18,6 +18,10 @@ RUN apt-get update -y \ COPY * /usr/local/bin/ RUN mv /usr/local/bin/librocksdb.so* /usr/local/lib/ && ldconfig +# Arize - remove a few base utilities flagged by security scans as having suid/sgid set. +# Note: we did not see these bits set ourselves when deploying in our test cluster. +RUN rm -f /usr/bin/mount /usr/bin/umount /usr/bin/su /usr/bin/wall + # Run as non-privileged "gazette" user. RUN useradd gazette --create-home --shell /usr/sbin/nologin USER gazette diff --git a/mk/common-build.mk b/mk/common-build.mk index a0d1af77..32eeb20b 100644 --- a/mk/common-build.mk +++ b/mk/common-build.mk @@ -80,6 +80,7 @@ ci-release-%: $(ROCKSDIR)/librocksdb.so go-install $$($$@-targets) ln ${$@-targets} ${ROCKSDIR}/librocksdb.so.${ROCKSDB_VERSION} \ ${WORKDIR}/ci-release docker build \ + --no-cache \ -f ${COREDIR}/mk/ci-release.Dockerfile \ -t $(subst -,/,$*):latest \ ${WORKDIR}/ci-release/