From a7a4fcef9d984d88e81998022a57898d88711e76 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 15 Feb 2023 02:41:18 +0000 Subject: [PATCH 1/6] build(deps): bump github.com/prometheus/client_golang Bumps [github.com/prometheus/client_golang](https://github.com/prometheus/client_golang) from 1.11.0 to 1.11.1. - [Release notes](https://github.com/prometheus/client_golang/releases) - [Changelog](https://github.com/prometheus/client_golang/blob/main/CHANGELOG.md) - [Commits](https://github.com/prometheus/client_golang/compare/v1.11.0...v1.11.1) --- updated-dependencies: - dependency-name: github.com/prometheus/client_golang dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 8d1e18c8..2deed072 100644 --- a/go.mod +++ b/go.mod @@ -30,7 +30,7 @@ require ( github.com/mattn/go-sqlite3 v2.0.3+incompatible github.com/olekukonko/tablewriter v0.0.5 github.com/pkg/errors v0.9.1 - github.com/prometheus/client_golang v1.11.0 + github.com/prometheus/client_golang v1.11.1 github.com/prometheus/client_model v0.2.0 github.com/prometheus/common v0.30.0 // indirect github.com/prometheus/procfs v0.7.3 // indirect @@ -52,7 +52,7 @@ require ( golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e // indirect google.golang.org/api v0.56.0 google.golang.org/grpc v1.40.0 - google.golang.org/protobuf v1.27.1 // indirect + google.golang.org/protobuf v1.27.1 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c gopkg.in/yaml.v2 v2.4.0 k8s.io/api v0.0.0-20190620073856-dcce3486da33 diff --git a/go.sum b/go.sum index 98c567d9..bc0ffe6c 100644 --- a/go.sum +++ b/go.sum @@ -154,8 +154,6 @@ github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4er github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= @@ -328,8 +326,9 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_golang v1.11.1 h1:+4eQaD7vAZ6DsfsxB15hbE0odUjGI5ARs9yskGu1v4s= +github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= From 9a244c0467dbda757c597ebfdb0c78ffd210a339 Mon Sep 17 00:00:00 2001 From: Darren Dowker Date: Mon, 13 Mar 2023 09:20:22 -0700 Subject: [PATCH 2/6] align with grpc base/balancer to trigger reconnect in Idle state --- broker/protocol/dispatcher.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/broker/protocol/dispatcher.go b/broker/protocol/dispatcher.go index e6fb35ae..46d73e92 100644 --- a/broker/protocol/dispatcher.go +++ b/broker/protocol/dispatcher.go @@ -117,12 +117,19 @@ func (d *dispatcher) UpdateSubConnState(sc balancer.SubConn, state balancer.SubC panic("unexpected SubConn") } - if state.ConnectivityState == connectivity.Connecting && d.connState[sc] == connectivity.TransientFailure { + if (state.ConnectivityState == connectivity.Connecting || state.ConnectivityState == connectivity.Idle) && + d.connState[sc] == connectivity.TransientFailure { // gRPC will quickly transition failed connections back into a Connecting // state. In many cases, such as a remote-initiated close from a // shutting-down server, the SubConn may never return. Until we see a // successful re-connect, continue to consider the SubConn as broken // (and trigger invalidations of cached Routes which use it). + + if state.ConnectivityState == connectivity.Idle { + sc.Connect() + } + d.mu.Unlock() + return } else { d.connState[sc] = state.ConnectivityState } @@ -132,6 +139,10 @@ func (d *dispatcher) UpdateSubConnState(sc balancer.SubConn, state balancer.SubC delete(d.connID, sc) delete(d.connState, sc) } + + if state.ConnectivityState == connectivity.Idle { + sc.Connect() + } d.mu.Unlock() // Notify gRPC that block requests may now be able to proceed. From 2c8515c7210f088e2f3e182aa4ef64b30dea447c Mon Sep 17 00:00:00 2001 From: Harjit Singh Date: Mon, 31 Oct 2022 10:33:17 -0700 Subject: [PATCH 3/6] Fix for sts:AssumeRoleWithWebIdentity, #16048 --- broker/fragment/store_s3.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/broker/fragment/store_s3.go b/broker/fragment/store_s3.go index 1a700f03..4488625f 100644 --- a/broker/fragment/store_s3.go +++ b/broker/fragment/store_s3.go @@ -220,7 +220,6 @@ func (s *s3Backend) s3Client(ep *url.URL) (cfg S3StoreConfig, client *s3.S3, err } awsSession, err := session.NewSessionWithOptions(session.Options{ - Config: *awsConfig, Profile: cfg.Profile, }) if err != nil { @@ -250,7 +249,7 @@ func (s *s3Backend) s3Client(ep *url.URL) (cfg S3StoreConfig, client *s3.S3, err "providerName": creds.ProviderName, }).Info("constructed new aws.Session") - client = s3.New(awsSession) + client = s3.New(awsSession, awsConfig) s.clients[key] = client return From 1545ac3ac70c7ee3af84a0faa7c34df99b4b0388 Mon Sep 17 00:00:00 2001 From: ddowker Date: Mon, 25 Sep 2023 11:06:53 -0700 Subject: [PATCH 4/6] Arize get rid of ebay dev/suid guid CVEs (#17) --- mk/ci-release.Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mk/ci-release.Dockerfile b/mk/ci-release.Dockerfile index 02365593..3853d155 100644 --- a/mk/ci-release.Dockerfile +++ b/mk/ci-release.Dockerfile @@ -18,6 +18,10 @@ RUN apt-get update -y \ COPY * /usr/local/bin/ RUN mv /usr/local/bin/librocksdb.so* /usr/local/lib/ && ldconfig +# Arize - remove a few base utilities flagged by security scans as having suid/sgid set. +# Note: we did not see these bits set ourselves when deploying in our test cluster. +RUN rm -f /usr/bin/mount /usr/bin/umount /usr/bin/su /usr/bin/wall + # Run as non-privileged "gazette" user. RUN useradd gazette --create-home --shell /usr/sbin/nologin USER gazette From d6b858fc9b507f7468c26205298a54210b37f09e Mon Sep 17 00:00:00 2001 From: Darren Dowker Date: Wed, 8 Nov 2023 10:03:10 -0800 Subject: [PATCH 5/6] change ci to match our mirror settings --- .github/workflows/ci-workflow.yaml | 48 +++++++++++++++++++++++++----- mk/common-build.mk | 1 + 2 files changed, 41 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci-workflow.yaml b/.github/workflows/ci-workflow.yaml index 3442df82..0ee1ac26 100644 --- a/.github/workflows/ci-workflow.yaml +++ b/.github/workflows/ci-workflow.yaml @@ -3,11 +3,14 @@ name: Gazette Continuous Integration # We build on any push to a branch, or when a release is created. on: pull_request: + branches: + - "arize" paths-ignore: - "docs/**" push: branches: - - "master" + - "arize" + - "arize-dev/*" # Ignore pushes to tags, since those ought to be handled by the release created event. tags-ignore: - "*" @@ -17,6 +20,7 @@ on: # Without this additional restriction, GH actions will trigger multiple runs for a single # release, because it fires off separate events creating vs publishing the release. types: [created] + workflow_dispatch: env: # This is only used as the cache key to prevent rebuilding rocksdb every time. Eventually @@ -24,6 +28,10 @@ env: # For now, ensure that it's changed both here and in mk/common-config.mk. ROCKSDB_VERSION: "6.22.1" +permissions: + id-token: write + contents: read + jobs: build: name: "Build" @@ -56,24 +64,33 @@ jobs: fi else # This is not a release, so we'll use 'dev-' for the version number - # and just 'latest-dev' for the docker tag. + # and just 'latest' for the docker tag. sha=${{ github.sha }} version="dev-${sha:0:7}" # If this is a master build, then we'll treat this as a release and just use the # hard-coded tag as the docker image tag. if [[ '${{ github.ref }}' == 'refs/heads/master' ]]; then # We don't want to put the git sha in the docker tag because otherwise they'll - # accumulate forever and just clutter up the page on docker hub. So 'latest-dev' + # accumulate forever and just clutter up the page on docker hub. So 'latest' # just always gets you the most recent master build, and if you want a specific master # build, then you can use the '@sha256:...' syntax. - docker_tag="latest-dev" + docker_tag="${{ secrets.REGISTRY_PATH }}/gazette/broker:latest" + push_images='true' + elif [[ '${{ github.ref }}' == 'refs/heads/arize' ]]; then + version="0.89.1-arize-${sha:0:7}" + docker_tag="${{ secrets.REGISTRY_PATH }}/gazette/broker:arize-${sha:0:7}" + push_images='true' + elif [[ '${{ github.ref }}' == *'arize'* ]]; then + version="0.89.1-dev-${sha:0:7}" + docker_tag="${{ secrets.REGISTRY_PATH }}/gazette/broker:dev-${sha:0:7}" push_images='true' else + docker_tag="latest" push_images='false' fi fi echo ::set-output name=VERSION::${version} - echo ::set-output name=DOCKER_TAG::${docker_tag:-$version} + echo ::set-output name=DOCKER_TAG::${docker_tag} echo ::set-output name=PUSH_IMAGES::${push_images} echo ::set-output name=IS_RELEASE::${is_release} @@ -105,6 +122,20 @@ jobs: # because go will use its own finer-grained cache invalidation logic. restore-keys: "go-mod-c4-" + + - uses: 'google-github-actions/auth@v1' + with: + token_format: "access_token" + project_id: ${{ secrets.PROJECT_ID }} + workload_identity_provider: projects/${{ secrets.PROJECT_NUMBER }}/locations/global/workloadIdentityPools/github/providers/github-actions + service_account: ${{ secrets.SERVICE_ACCOUNT }} + + - name: 'Set up Cloud SDK' + uses: 'google-github-actions/setup-gcloud@v1' + + - name: 'Use gcloud CLI' + run: gcloud info + - name: "Build Binaries" run: "make as-ci target=release-linux-binaries VERSION=${{ steps.release_info.outputs.VERSION }}" @@ -130,10 +161,11 @@ jobs: upload_url: "${{ github.event.release.upload_url }}" asset_content_type: application/zip + - name: "Build and Push Docker Images" if: steps.release_info.outputs.PUSH_IMAGES == 'true' run: | - docker login -u '${{ secrets.DOCKER_USERNAME }}' -p '${{ secrets.DOCKER_PASSWORD }}' ${{ secrets.DOCKER_REGISTRY }} - make as-ci target=ci-release-gazette-examples VERSION=${{ steps.release_info.outputs.VERSION }} make as-ci target=ci-release-gazette-broker VERSION=${{ steps.release_info.outputs.VERSION }} - make push-to-registry REGISTRY=${{ secrets.DOCKER_REGISTRY }} RELEASE_TAG=${{ steps.release_info.outputs.DOCKER_TAG }} + docker tag gazette/broker:latest ${{ steps.release_info.outputs.DOCKER_TAG }} + gcloud auth configure-docker ${{ secrets.REGISTRY }} + docker push ${{ steps.release_info.outputs.DOCKER_TAG }} diff --git a/mk/common-build.mk b/mk/common-build.mk index a0d1af77..32eeb20b 100644 --- a/mk/common-build.mk +++ b/mk/common-build.mk @@ -80,6 +80,7 @@ ci-release-%: $(ROCKSDIR)/librocksdb.so go-install $$($$@-targets) ln ${$@-targets} ${ROCKSDIR}/librocksdb.so.${ROCKSDB_VERSION} \ ${WORKDIR}/ci-release docker build \ + --no-cache \ -f ${COREDIR}/mk/ci-release.Dockerfile \ -t $(subst -,/,$*):latest \ ${WORKDIR}/ci-release/ From 3ba90ac7787ffdc0cd4066ed390746b8acc84c18 Mon Sep 17 00:00:00 2001 From: ddowker Date: Wed, 29 Nov 2023 15:39:10 -0800 Subject: [PATCH 6/6] Support shared key credentials with blob SAS in latest store_azure.go (#2) --- broker/fragment/store_azure.go | 58 +++++++++++++++++++++++++--------- 1 file changed, 43 insertions(+), 15 deletions(-) diff --git a/broker/fragment/store_azure.go b/broker/fragment/store_azure.go index cc55005f..e97ebd9d 100644 --- a/broker/fragment/store_azure.go +++ b/broker/fragment/store_azure.go @@ -48,6 +48,8 @@ type azureBackend struct { clientMu sync.Mutex udc *service.UserDelegationCredential udcExp *time.Time + + sharedKeyCredentials *sas.SharedKeyCredential } func (a *azureBackend) Provider() string { @@ -57,31 +59,56 @@ func (a *azureBackend) Provider() string { // See here for an example of how to use the Azure client libraries to create signatures: // https://github.com/Azure/azure-sdk-for-go/blob/main/sdk/storage/azblob/service/examples_test.go#L285 func (a *azureBackend) SignGet(ep *url.URL, fragment pb.Fragment, d time.Duration) (string, error) { + var ( + sasQueryParams sas.QueryParameters + err error + ) + cfg, _, err := a.azureClient(ep) if err != nil { return "", err } blobName := cfg.rewritePath(cfg.prefix, fragment.ContentPath()) - udc, err := a.getUserDelegationCredential() - if err != nil { - return "", err - } + if ep.Scheme == "azure" { + // Note: for arize we assume azure scheme is for blob SAS (as opposed to container SAS in azure-ad case) + perms := sas.BlobPermissions{Add: true, Read: true, Write: true} - sasQueryParams, err := sas.BlobSignatureValues{ - Protocol: sas.ProtocolHTTPS, // Users MUST use HTTPS (not HTTP) - ExpiryTime: time.Now().UTC().Add(d), // Timestamps are expected in UTC https://docs.microsoft.com/en-us/rest/api/storageservices/create-service-sas#service-sas-example - ContainerName: cfg.containerName, - BlobName: blobName, + sasQueryParams, err = sas.BlobSignatureValues{ + Protocol: sas.ProtocolHTTPS, // Users MUST use HTTPS (not HTTP) + ExpiryTime: time.Now().UTC().Add(d), + ContainerName: cfg.containerName, + BlobName: blobName, + Permissions: perms.String(), + }.SignWithSharedKey(a.sharedKeyCredentials) - // To produce a container SAS (as opposed to a blob SAS), assign to Permissions using - // ContainerSASPermissions and make sure the BlobName field is "" (the default). - Permissions: to.Ptr(sas.ContainerPermissions{Read: true, Add: true, Write: true}).String(), - }.SignWithUserDelegation(udc) + if err != nil { + return "", err + } + } else if ep.Scheme == "azure-ad" { + udc, err := a.getUserDelegationCredential() + if err != nil { + return "", err + } - if err != nil { - return "", err + sasQueryParams, err = sas.BlobSignatureValues{ + Protocol: sas.ProtocolHTTPS, // Users MUST use HTTPS (not HTTP) + ExpiryTime: time.Now().UTC().Add(d), // Timestamps are expected in UTC https://docs.microsoft.com/en-us/rest/api/storageservices/create-service-sas#service-sas-example + ContainerName: cfg.containerName, + BlobName: blobName, + + // To produce a container SAS (as opposed to a blob SAS), assign to Permissions using + // ContainerSASPermissions and make sure the BlobName field is "" (the default). + Permissions: to.Ptr(sas.ContainerPermissions{Read: true, Add: true, Write: true}).String(), + }.SignWithUserDelegation(udc) + + if err != nil { + return "", err + } + } else { + return "", fmt.Errorf("unknown scheme: %s", ep.Scheme) } + return fmt.Sprintf("%s/%s?%s", cfg.containerURL(), blobName, sasQueryParams.Encode()), nil } @@ -258,6 +285,7 @@ func (a *azureBackend) azureClient(ep *url.URL) (cfg AzureStoreConfig, client pi if err != nil { return cfg, nil, err } + a.sharedKeyCredentials = sharedKeyCred // Arize addition serviceClient, err := service.NewClientWithSharedKeyCredential(cfg.serviceUrl(), sharedKeyCred, &service.ClientOptions{}) if err != nil { return cfg, nil, err