From a1d8ded473745e30d607692fede487db8b465c8f Mon Sep 17 00:00:00 2001 From: Haytham AbuelFutuh Date: Wed, 25 Nov 2020 12:26:21 -0800 Subject: [PATCH 01/22] Update kind version to address github vulnerability --- .github/workflows/master.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 0c3db359d..75838371e 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -48,7 +48,7 @@ jobs: key: /tmp/docker-images-${{ github.event.after }} - name: Prime docker cache run: docker load -i /tmp/tmp/docker-images/snapshot.tar || true - - uses: engineerd/setup-kind@v0.4.0 + - uses: engineerd/setup-kind@v0.5.0 - name: End2End run: | kubectl cluster-info From 10673982008cdf03ab7dbab7687eeb7d3b1ff7c7 Mon Sep 17 00:00:00 2001 From: Haytham AbuelFutuh Date: Mon, 30 Nov 2020 13:19:09 -0800 Subject: [PATCH 02/22] Push to Container Registry (#143) * Migrate to github container registry * Fix variable name --- .github/workflows/master.yml | 6 +++--- .github/workflows/pull_request.yml | 4 ++-- boilerplate/lyft/github_workflows/master.yml | 8 ++++---- boilerplate/lyft/github_workflows/pull_request.yml | 4 ++-- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 75838371e..186606745 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -107,12 +107,12 @@ jobs: - name: Push Docker Image to Github Registry uses: whoan/docker-build-with-cache-action@v5 with: - username: "${{ github.actor }}" - password: "${{ secrets.GITHUB_TOKEN }}" + username: "${{ secrets.PUBLISH_CONTAINER_ACTOR }}" + password: "${{ secrets.PUBLIC_CONTAINER }}" image_name: ${{ github.repository }}/${{ github.event.repository.name }} image_tag: latest,${{ github.sha }},${{ needs.bump-version.outputs.version }} push_git_tag: true - registry: docker.pkg.github.com + registry: ghcr.io build_extra_args: "--compress=true" push-dockerhub: diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index a51d93175..1f6448c85 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -44,7 +44,7 @@ jobs: key: /tmp/docker-images-${{ github.event.after }} - name: Prime docker cache run: docker load -i /tmp/tmp/docker-images/snapshot.tar || true - - uses: engineerd/setup-kind@v0.4.0 + - uses: engineerd/setup-kind@v0.5.0 - name: End2End run: | kubectl cluster-info @@ -68,7 +68,7 @@ jobs: key: /tmp/docker-images-${{ github.event.after }} - name: Prime docker cache run: docker load -i /tmp/tmp/docker-images/snapshot-builder.tar || true - - uses: engineerd/setup-kind@v0.4.0 + - uses: engineerd/setup-kind@v0.5.0 - name: Integration run: | kubectl cluster-info diff --git a/boilerplate/lyft/github_workflows/master.yml b/boilerplate/lyft/github_workflows/master.yml index 0c3db359d..186606745 100644 --- a/boilerplate/lyft/github_workflows/master.yml +++ b/boilerplate/lyft/github_workflows/master.yml @@ -48,7 +48,7 @@ jobs: key: /tmp/docker-images-${{ github.event.after }} - name: Prime docker cache run: docker load -i /tmp/tmp/docker-images/snapshot.tar || true - - uses: engineerd/setup-kind@v0.4.0 + - uses: engineerd/setup-kind@v0.5.0 - name: End2End run: | kubectl cluster-info @@ -107,12 +107,12 @@ jobs: - name: Push Docker Image to Github Registry uses: whoan/docker-build-with-cache-action@v5 with: - username: "${{ github.actor }}" - password: "${{ secrets.GITHUB_TOKEN }}" + username: "${{ secrets.PUBLISH_CONTAINER_ACTOR }}" + password: "${{ secrets.PUBLIC_CONTAINER }}" image_name: ${{ github.repository }}/${{ github.event.repository.name }} image_tag: latest,${{ github.sha }},${{ needs.bump-version.outputs.version }} push_git_tag: true - registry: docker.pkg.github.com + registry: ghcr.io build_extra_args: "--compress=true" push-dockerhub: diff --git a/boilerplate/lyft/github_workflows/pull_request.yml b/boilerplate/lyft/github_workflows/pull_request.yml index a51d93175..1f6448c85 100644 --- a/boilerplate/lyft/github_workflows/pull_request.yml +++ b/boilerplate/lyft/github_workflows/pull_request.yml @@ -44,7 +44,7 @@ jobs: key: /tmp/docker-images-${{ github.event.after }} - name: Prime docker cache run: docker load -i /tmp/tmp/docker-images/snapshot.tar || true - - uses: engineerd/setup-kind@v0.4.0 + - uses: engineerd/setup-kind@v0.5.0 - name: End2End run: | kubectl cluster-info @@ -68,7 +68,7 @@ jobs: key: /tmp/docker-images-${{ github.event.after }} - name: Prime docker cache run: docker load -i /tmp/tmp/docker-images/snapshot-builder.tar || true - - uses: engineerd/setup-kind@v0.4.0 + - uses: engineerd/setup-kind@v0.5.0 - name: Integration run: | kubectl cluster-info From 5f7732767e299e1a5e492bfd6aa010abbdab9d7e Mon Sep 17 00:00:00 2001 From: Haytham AbuelFutuh Date: Mon, 30 Nov 2020 14:30:36 -0800 Subject: [PATCH 03/22] Typo in workflow --- .github/workflows/master.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 186606745..b8c9ad2e6 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -108,7 +108,7 @@ jobs: uses: whoan/docker-build-with-cache-action@v5 with: username: "${{ secrets.PUBLISH_CONTAINER_ACTOR }}" - password: "${{ secrets.PUBLIC_CONTAINER }}" + password: "${{ secrets.PUBLISH_CONTAINER }}" image_name: ${{ github.repository }}/${{ github.event.repository.name }} image_tag: latest,${{ github.sha }},${{ needs.bump-version.outputs.version }} push_git_tag: true From efb9474e2fe69615a38780c9ff449294df225b80 Mon Sep 17 00:00:00 2001 From: Katrina Rogan Date: Mon, 30 Nov 2020 14:33:25 -0800 Subject: [PATCH 04/22] Add matchable plugin overrides for single task executions too (#144) --- pkg/manager/impl/execution_manager.go | 22 ++++++++++++++----- pkg/manager/impl/execution_manager_test.go | 10 ++++----- pkg/workflowengine/impl/propeller_executor.go | 1 + pkg/workflowengine/interfaces/executor.go | 19 ++++++++-------- 4 files changed, 32 insertions(+), 20 deletions(-) diff --git a/pkg/manager/impl/execution_manager.go b/pkg/manager/impl/execution_manager.go index b22809d08..dfe85367e 100644 --- a/pkg/manager/impl/execution_manager.go +++ b/pkg/manager/impl/execution_manager.go @@ -174,7 +174,7 @@ func (m *ExecutionManager) addLabelsAndAnnotations(requestSpec *admin.ExecutionS } func (m *ExecutionManager) addPluginOverrides(ctx context.Context, executionID *core.WorkflowExecutionIdentifier, - workflowName, launchPlanName string, partiallyPopulatedInputs *workflowengineInterfaces.ExecuteWorkflowInput) error { + workflowName, launchPlanName string) ([]*admin.PluginOverride, error) { override, err := m.resourceManager.GetResource(ctx, interfaces.ResourceRequest{ Project: executionID.Project, Domain: executionID.Domain, @@ -185,13 +185,13 @@ func (m *ExecutionManager) addPluginOverrides(ctx context.Context, executionID * if err != nil { ec, ok := err.(errors.FlyteAdminError) if !ok || ec.Code() != codes.NotFound { - return err + return nil, err } } if override != nil && override.Attributes != nil && override.Attributes.GetPluginOverrides() != nil { - partiallyPopulatedInputs.TaskPluginOverrides = override.Attributes.GetPluginOverrides().Overrides + return override.Attributes.GetPluginOverrides().Overrides, nil } - return nil + return nil, nil } func (m *ExecutionManager) offloadInputs(ctx context.Context, literalMap *core.LiteralMap, identifier *core.WorkflowExecutionIdentifier, key string) (storage.DataReference, error) { @@ -476,6 +476,14 @@ func (m *ExecutionManager) launchSingleTaskExecution( executeTaskInputs.Annotations = request.Spec.Annotations.Values } + overrides, err := m.addPluginOverrides(ctx, &workflowExecutionID, workflowExecutionID.Name, "") + if err != nil { + return nil, nil, err + } + if overrides != nil { + executeTaskInputs.TaskPluginOverrides = overrides + } + execInfo, err := m.workflowExecutor.ExecuteTask(ctx, executeTaskInputs) if err != nil { m.systemMetrics.PropellerFailures.Inc() @@ -642,11 +650,13 @@ func (m *ExecutionManager) launchExecutionAndPrepareModel( return nil, nil, err } - err = m.addPluginOverrides(ctx, &workflowExecutionID, launchPlan.GetSpec().WorkflowId.Name, launchPlan.Id.Name, - &executeWorkflowInputs) + overrides, err := m.addPluginOverrides(ctx, &workflowExecutionID, launchPlan.GetSpec().WorkflowId.Name, launchPlan.Id.Name) if err != nil { return nil, nil, err } + if overrides != nil { + executeWorkflowInputs.TaskPluginOverrides = overrides + } execInfo, err := m.workflowExecutor.ExecuteWorkflow(ctx, executeWorkflowInputs) if err != nil { diff --git a/pkg/manager/impl/execution_manager_test.go b/pkg/manager/impl/execution_manager_test.go index 42b3ed63f..f0e5d85ba 100644 --- a/pkg/manager/impl/execution_manager_test.go +++ b/pkg/manager/impl/execution_manager_test.go @@ -2157,10 +2157,10 @@ func TestAddPluginOverrides(t *testing.T) { db, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil) - err := execManager.(*ExecutionManager).addPluginOverrides( - context.Background(), executionID, workflowName, launchPlanName, &partiallyPopulatedInputs) + taskPluginOverrides, err := execManager.(*ExecutionManager).addPluginOverrides( + context.Background(), executionID, workflowName, launchPlanName) assert.NoError(t, err) - assert.Len(t, partiallyPopulatedInputs.TaskPluginOverrides, 2) + assert.Len(t, taskPluginOverrides, 2) for _, override := range partiallyPopulatedInputs.TaskPluginOverrides { if override.TaskType == "python" { assert.EqualValues(t, []string{"plugin a"}, override.PluginId) @@ -2190,8 +2190,8 @@ func TestPluginOverrides_ResourceGetFailure(t *testing.T) { db, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil) - err := execManager.(*ExecutionManager).addPluginOverrides( - context.Background(), executionID, workflowName, launchPlanName, &workflowengineInterfaces.ExecuteWorkflowInput{}) + _, err := execManager.(*ExecutionManager).addPluginOverrides( + context.Background(), executionID, workflowName, launchPlanName) assert.Error(t, err, "uh oh") } diff --git a/pkg/workflowengine/impl/propeller_executor.go b/pkg/workflowengine/impl/propeller_executor.go index 4d41ae4c8..c8657c322 100644 --- a/pkg/workflowengine/impl/propeller_executor.go +++ b/pkg/workflowengine/impl/propeller_executor.go @@ -225,6 +225,7 @@ func (c *FlytePropeller) ExecuteTask(ctx context.Context, input interfaces.Execu flyteWf.Labels = labels annotations := addMapValues(input.Annotations, flyteWf.Annotations) flyteWf.Annotations = annotations + addExecutionOverrides(input.TaskPluginOverrides, flyteWf) /* TODO(katrogan): uncomment once propeller has updated the flyte workflow CRD. diff --git a/pkg/workflowengine/interfaces/executor.go b/pkg/workflowengine/interfaces/executor.go index bd468ba28..93174f557 100644 --- a/pkg/workflowengine/interfaces/executor.go +++ b/pkg/workflowengine/interfaces/executor.go @@ -22,15 +22,16 @@ type ExecuteWorkflowInput struct { } type ExecuteTaskInput struct { - ExecutionID *core.WorkflowExecutionIdentifier - WfClosure core.CompiledWorkflowClosure - Inputs *core.LiteralMap - ReferenceName string - Auth *admin.AuthRole - AcceptedAt time.Time - Labels map[string]string - Annotations map[string]string - QueueingBudget time.Duration + ExecutionID *core.WorkflowExecutionIdentifier + WfClosure core.CompiledWorkflowClosure + Inputs *core.LiteralMap + ReferenceName string + Auth *admin.AuthRole + AcceptedAt time.Time + Labels map[string]string + Annotations map[string]string + QueueingBudget time.Duration + TaskPluginOverrides []*admin.PluginOverride } type TerminateWorkflowInput struct { From 803dcfb818ba27674efc2b16115e99e7dd535db4 Mon Sep 17 00:00:00 2001 From: Haytham AbuelFutuh Date: Mon, 30 Nov 2020 15:13:28 -0800 Subject: [PATCH 05/22] Link docker image to repo --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index 9d3d17eff..5850f983e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,6 +25,8 @@ ENV PATH="/artifacts:${PATH}" # This will eventually move to centurylink/ca-certs:latest for minimum possible image size FROM alpine:3.10 +LABEL org.opencontainers.image.source https://github.com/lyft/flyteadmin + COPY --from=builder /artifacts /bin # Ensure the latest CA certs are present to authenticate SSL connections. From fdb5763266f12a22a47902497da4a839f16e929f Mon Sep 17 00:00:00 2001 From: Haytham AbuelFutuh Date: Mon, 30 Nov 2020 15:14:01 -0800 Subject: [PATCH 06/22] Shorten ghcr image name --- .github/workflows/master.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index b8c9ad2e6..49e25a6dc 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -109,7 +109,7 @@ jobs: with: username: "${{ secrets.PUBLISH_CONTAINER_ACTOR }}" password: "${{ secrets.PUBLISH_CONTAINER }}" - image_name: ${{ github.repository }}/${{ github.event.repository.name }} + image_name: ${{ github.repository }} image_tag: latest,${{ github.sha }},${{ needs.bump-version.outputs.version }} push_git_tag: true registry: ghcr.io From 5ab022548c9a119e99ff60b7a94284a884cb05db Mon Sep 17 00:00:00 2001 From: Haytham AbuelFutuh Date: Mon, 30 Nov 2020 16:24:58 -0800 Subject: [PATCH 07/22] Add badges --- README.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.rst b/README.rst index 6346aa74f..81961ea03 100644 --- a/README.rst +++ b/README.rst @@ -1,5 +1,13 @@ Flyteadmin ============= +[![Current Release](https://img.shields.io/github/release/lyft/flyteadmin.svg)](https://github.com/lyft/flyteadmin/releases/latest) +![Master](https://github.com/lyft/flyteadmin/workflows/Master/badge.svg) +[![GoDoc](https://godoc.org/github.com/lyft/flyteadmin?status.svg)](https://pkg.go.dev/mod/github.com/lyft/flyteadmin) +[![License](https://img.shields.io/badge/LICENSE-Apache2.0-ff69b4.svg)](http://www.apache.org/licenses/LICENSE-2.0.html) +[![CodeCoverage](https://img.shields.io/codecov/c/github/lyft/flyteadmin.svg)](https://codecov.io/gh/lyft/flyteadmin) +[![Go Report Card](https://goreportcard.com/badge/github.com/lyft/flyteadmin)](https://goreportcard.com/report/github.com/lyft/flyteadmin) +![Commit activity](https://img.shields.io/github/commit-activity/w/lyft/flyteadmin.svg?style=plastic) +![Commit since last release](https://img.shields.io/github/commits-since/lyft/flyteadmin/latest.svg?style=plastic) Flyteadmin is the control plane for Flyte responsible for managing entities (task, workflows, launch plans) and administering workflow executions. Flyteadmin implements the From 42933ceab1b202b803a9ef11a46fe5a9292cccf2 Mon Sep 17 00:00:00 2001 From: Haytham AbuelFutuh Date: Mon, 30 Nov 2020 16:25:41 -0800 Subject: [PATCH 08/22] Rename README.rst to README.md --- README.rst => README.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename README.rst => README.md (100%) diff --git a/README.rst b/README.md similarity index 100% rename from README.rst rename to README.md From b0bba8de9cb3a2fbc3192492e5952caf00c02bab Mon Sep 17 00:00:00 2001 From: Haytham AbuelFutuh Date: Mon, 30 Nov 2020 16:30:21 -0800 Subject: [PATCH 09/22] Update README.md --- README.md | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 81961ea03..7ad61ebd9 100644 --- a/README.md +++ b/README.md @@ -11,9 +11,9 @@ Flyteadmin Flyteadmin is the control plane for Flyte responsible for managing entities (task, workflows, launch plans) and administering workflow executions. Flyteadmin implements the -`AdminService `_ which +[AdminService](https://github.com/lyft/flyteidl/blob/master/protos/flyteidl/service/admin.proto) which defines a stateless REST/gRPC service for interacting with registered Flyte entities and executions. -Flyteadmin uses a relational style Metadata Store abstracted by `GORM `_ ORM library. +Flyteadmin uses a relational style Metadata Store abstracted by [GORM](http://gorm.io/) ORM library. Before Check-In ~~~~~~~~~~~~~~~ @@ -21,14 +21,14 @@ Before Check-In Flyte Admin has a few useful make targets for linting and testing. Please use these before checking in to help suss out minor bugs and linting errors. - .. code-block:: console +``` + $ make goimports +``` - make goimports +``` + $ make test_unit +``` - .. code-block:: console - - make test_unit - - .. code-block:: console - - make lint +``` + $ make lint +``` From fc1dd186c2463d693d9cf1e3526bad5102b36eb3 Mon Sep 17 00:00:00 2001 From: Haytham AbuelFutuh Date: Mon, 30 Nov 2020 16:30:38 -0800 Subject: [PATCH 10/22] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7ad61ebd9..4f4fdd762 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ defines a stateless REST/gRPC service for interacting with registered Flyte enti Flyteadmin uses a relational style Metadata Store abstracted by [GORM](http://gorm.io/) ORM library. Before Check-In -~~~~~~~~~~~~~~~ +--------------- Flyte Admin has a few useful make targets for linting and testing. Please use these before checking in to help suss out minor bugs and linting errors. From 8a4e19993720fe406397595cd308e6e0ecda2c07 Mon Sep 17 00:00:00 2001 From: Honnix Date: Tue, 1 Dec 2020 23:03:13 +0100 Subject: [PATCH 11/22] Add filter and sort param when listing projects (#142) * Add filter and sort param when listing projects --- go.mod | 3 +- go.sum | 4 +- pkg/clusterresource/controller.go | 10 +++- pkg/common/entity.go | 1 + pkg/manager/impl/project_manager.go | 44 +++++++++++++- pkg/manager/impl/project_manager_test.go | 41 +++++++++++-- pkg/manager/impl/util/filters.go | 2 + pkg/repositories/gormimpl/project_repo.go | 33 +++++++++-- .../gormimpl/project_repo_test.go | 57 ++++++++++++------- pkg/repositories/interfaces/project_repo.go | 5 +- pkg/repositories/mocks/project_repo.go | 7 +-- 11 files changed, 164 insertions(+), 43 deletions(-) diff --git a/go.mod b/go.mod index 324a079e9..129310191 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/jinzhu/gorm v1.9.12 github.com/kelseyhightower/envconfig v1.4.0 // indirect github.com/lib/pq v1.3.0 - github.com/lyft/flyteidl v0.18.10 + github.com/lyft/flyteidl v0.18.11 github.com/lyft/flytepropeller v0.3.17 github.com/lyft/flytestdlib v0.3.9 github.com/magiconair/properties v1.8.1 @@ -58,5 +58,4 @@ replace ( k8s.io/api => github.com/lyft/api v0.0.0-20191031200350-b49a72c274e0 k8s.io/apimachinery => github.com/lyft/apimachinery v0.0.0-20191031200210-047e3ea32d7f k8s.io/client-go => k8s.io/client-go v0.0.0-20191016111102-bec269661e48 - ) diff --git a/go.sum b/go.sum index 3461b942f..3d1e0d901 100644 --- a/go.sum +++ b/go.sum @@ -464,8 +464,8 @@ github.com/lyft/flyteidl v0.17.0/go.mod h1:/zQXxuHO11u/saxTTZc8oYExIGEShXB+xCB1/ github.com/lyft/flyteidl v0.18.0/go.mod h1:/zQXxuHO11u/saxTTZc8oYExIGEShXB+xCB1/F1Cu20= github.com/lyft/flyteidl v0.18.6 h1:HGbxHI8avEDvoPqcO2+/BoJVcP9sjOj4qwJ/wNRWuoA= github.com/lyft/flyteidl v0.18.6/go.mod h1:/zQXxuHO11u/saxTTZc8oYExIGEShXB+xCB1/F1Cu20= -github.com/lyft/flyteidl v0.18.10 h1:kM7HKNsv0S9H0nJMPGdTbD53csjc7ueO9eXX+UtZ3fk= -github.com/lyft/flyteidl v0.18.10/go.mod h1:/zQXxuHO11u/saxTTZc8oYExIGEShXB+xCB1/F1Cu20= +github.com/lyft/flyteidl v0.18.11 h1:24NaFYWxANhRbwKfvkgu8axGTWUcl1tgZBqNJutKNJ8= +github.com/lyft/flyteidl v0.18.11/go.mod h1:/zQXxuHO11u/saxTTZc8oYExIGEShXB+xCB1/F1Cu20= github.com/lyft/flyteplugins v0.5.1/go.mod h1:8zhqFG9BzbHNQGEXzGYltTJLD+KTmQZkanxXgeFI25c= github.com/lyft/flytepropeller v0.3.17 h1:a2PVqWjnn8oNEeayAqNizMAtEixl/F3S4vd8z4kbiqI= github.com/lyft/flytepropeller v0.3.17/go.mod h1:T8Utxqv7B5USAX9c/Qh0lBbKXHFSgOwwaISOd9h36P4= diff --git a/pkg/clusterresource/controller.go b/pkg/clusterresource/controller.go index d007cb56c..89ff4657c 100644 --- a/pkg/clusterresource/controller.go +++ b/pkg/clusterresource/controller.go @@ -32,6 +32,7 @@ import ( "github.com/lyft/flyteadmin/pkg/errors" "github.com/lyft/flyteadmin/pkg/repositories" + repositoriesInterfaces "github.com/lyft/flyteadmin/pkg/repositories/interfaces" "github.com/lyft/flyteadmin/pkg/runtime" runtimeInterfaces "github.com/lyft/flyteadmin/pkg/runtime/interfaces" "github.com/lyft/flytestdlib/promutils" @@ -357,7 +358,14 @@ func (c *controller) Sync(ctx context.Context) error { logger.Debugf(ctx, "Running an invocation of ClusterResource Sync") // Prefer to sync projects most newly created to ensure their resources get created first when other resources exist. - projects, err := c.db.ProjectRepo().ListAll(ctx, descCreatedAtSortParam) + filter, err := common.NewSingleValueFilter(common.Project, common.NotEqual, "state", int32(admin.Project_ARCHIVED)) + if err != nil { + return err + } + projects, err := c.db.ProjectRepo().List(ctx, repositoriesInterfaces.ListResourceInput{ + SortParameter: descCreatedAtSortParam, + InlineFilters: []common.InlineFilter{filter}, + }) if err != nil { return err } diff --git a/pkg/common/entity.go b/pkg/common/entity.go index ef6b65a0b..71274064f 100644 --- a/pkg/common/entity.go +++ b/pkg/common/entity.go @@ -16,6 +16,7 @@ const ( Workflow = "w" NamedEntity = "nen" NamedEntityMetadata = "nem" + Project = "p" ) // ResourceTypeToEntity maps a resource type to an entity suitable for use with Database filters diff --git a/pkg/manager/impl/project_manager.go b/pkg/manager/impl/project_manager.go index 336760e70..ee9a03d5d 100644 --- a/pkg/manager/impl/project_manager.go +++ b/pkg/manager/impl/project_manager.go @@ -2,14 +2,19 @@ package impl import ( "context" + "strconv" "github.com/lyft/flyteadmin/pkg/common" + "github.com/lyft/flyteadmin/pkg/errors" + "github.com/lyft/flyteadmin/pkg/manager/impl/util" "github.com/lyft/flyteadmin/pkg/manager/impl/validation" "github.com/lyft/flyteadmin/pkg/manager/interfaces" "github.com/lyft/flyteadmin/pkg/repositories" + repoInterfaces "github.com/lyft/flyteadmin/pkg/repositories/interfaces" "github.com/lyft/flyteadmin/pkg/repositories/transformers" runtimeInterfaces "github.com/lyft/flyteadmin/pkg/runtime/interfaces" "github.com/lyft/flyteidl/gen/pb-go/flyteidl/admin" + "google.golang.org/grpc/codes" ) type ProjectManager struct { @@ -49,14 +54,51 @@ func (m *ProjectManager) getDomains() []*admin.Domain { } func (m *ProjectManager) ListProjects(ctx context.Context, request admin.ProjectListRequest) (*admin.Projects, error) { - projectModels, err := m.db.ProjectRepo().ListAll(ctx, alphabeticalSortParam) + spec := util.FilterSpec{ + RequestFilters: request.Filters, + } + filters, err := util.GetDbFilters(spec, common.Project) if err != nil { return nil, err } + var sortParameter common.SortParameter + if request.SortBy != nil { + sortParameter, err = common.NewSortParameter(*request.SortBy) + if err != nil { + return nil, err + } + } else { + sortParameter = alphabeticalSortParam + } + + offset, err := validation.ValidateToken(request.Token) + if err != nil { + return nil, errors.NewFlyteAdminErrorf(codes.InvalidArgument, + "invalid pagination token %s for ListProjects", request.Token) + } + + // And finally, query the database + listProjectsInput := repoInterfaces.ListResourceInput{ + Limit: int(request.Limit), + Offset: offset, + InlineFilters: filters, + SortParameter: sortParameter, + } + projectModels, err := m.db.ProjectRepo().List(ctx, listProjectsInput) + if err != nil { + return nil, err + } projects := transformers.FromProjectModels(projectModels, m.getDomains()) + + var token string + if len(projects) == int(request.Limit) { + token = strconv.Itoa(offset + len(projects)) + } + return &admin.Projects{ Projects: projects, + Token: token, }, nil } diff --git a/pkg/manager/impl/project_manager_test.go b/pkg/manager/impl/project_manager_test.go index db52400b3..c25307f81 100644 --- a/pkg/manager/impl/project_manager_test.go +++ b/pkg/manager/impl/project_manager_test.go @@ -5,10 +5,11 @@ import ( "errors" "testing" + "github.com/lyft/flyteadmin/pkg/common" "github.com/lyft/flyteadmin/pkg/manager/impl/shared" - "github.com/lyft/flyteadmin/pkg/common" "github.com/lyft/flyteadmin/pkg/manager/impl/testutils" + "github.com/lyft/flyteadmin/pkg/repositories/interfaces" repositoryMocks "github.com/lyft/flyteadmin/pkg/repositories/mocks" "github.com/lyft/flyteadmin/pkg/repositories/models" runtimeInterfaces "github.com/lyft/flyteadmin/pkg/runtime/interfaces" @@ -45,10 +46,15 @@ func getMockApplicationConfigForProjectManagerTest() runtimeInterfaces.Applicati return &mockApplicationConfig } -func TestListProjects(t *testing.T) { +func testListProjects(request admin.ProjectListRequest, token string, orderExpr string, queryExpr *common.GormQueryExpr, t *testing.T) { repository := repositoryMocks.NewMockRepository() repository.ProjectRepo().(*repositoryMocks.MockProjectRepo).ListProjectsFunction = func( - ctx context.Context, parameter common.SortParameter) ([]models.Project, error) { + ctx context.Context, input interfaces.ListResourceInput) ([]models.Project, error) { + if len(input.InlineFilters) != 0 { + q, _ := input.InlineFilters[0].GetGormQueryExpr() + assert.Equal(t, *queryExpr, q) + } + assert.Equal(t, orderExpr, input.SortParameter.GetGormOrderExpr()) activeState := int32(admin.Project_ACTIVE) return []models.Project{ { @@ -61,16 +67,43 @@ func TestListProjects(t *testing.T) { } projectManager := NewProjectManager(repository, mockProjectConfigProvider) - resp, err := projectManager.ListProjects(context.Background(), admin.ProjectListRequest{}) + resp, err := projectManager.ListProjects(context.Background(), request) assert.NoError(t, err) assert.Len(t, resp.Projects, 1) + assert.Equal(t, token, resp.GetToken()) assert.Len(t, resp.Projects[0].Domains, 4) for _, domain := range resp.Projects[0].Domains { assert.Contains(t, testDomainsForProjManager, domain.Id) } } +func TestListProjects_NoFilters_LimitOne(t *testing.T) { + testListProjects(admin.ProjectListRequest{ + Token: "1", + Limit: 1, + }, "2", "identifier asc", nil, t) +} + +func TestListProjects_HighLimit_SortBy_Filter(t *testing.T) { + testListProjects(admin.ProjectListRequest{ + Token: "1", + Limit: 999, + Filters: "eq(project.name,foo)", + SortBy: &admin.Sort{ + Key: "name", + Direction: admin.Sort_DESCENDING, + }, + }, "", "name desc", &common.GormQueryExpr{ + Query: "name = ?", + Args: "foo", + }, t) +} + +func TestListProjects_NoToken_NoLimit(t *testing.T) { + testListProjects(admin.ProjectListRequest{}, "", "identifier asc", nil, t) +} + func TestProjectManager_CreateProject(t *testing.T) { mockRepository := repositoryMocks.NewMockRepository() var createFuncCalled bool diff --git a/pkg/manager/impl/util/filters.go b/pkg/manager/impl/util/filters.go index 2a029b1b8..d983ed14b 100644 --- a/pkg/manager/impl/util/filters.go +++ b/pkg/manager/impl/util/filters.go @@ -60,6 +60,7 @@ var filterFieldEntityPrefix = map[string]common.Entity{ "task_execution": common.TaskExecution, "entities": common.NamedEntity, "named_entity_metadata": common.NamedEntityMetadata, + "project": common.Project, } func parseField(field string, primaryEntity common.Entity) (common.Entity, string) { @@ -188,6 +189,7 @@ func getIdentifierFilters(entity common.Entity, spec FilterSpec) ([]common.Inlin func AddRequestFilters(requestFilters string, primaryEntity common.Entity, existingFilters []common.InlineFilter) ( []common.InlineFilter, error) { + if requestFilters == "" { return existingFilters, nil } diff --git a/pkg/repositories/gormimpl/project_repo.go b/pkg/repositories/gormimpl/project_repo.go index 924ebd352..59d156f00 100644 --- a/pkg/repositories/gormimpl/project_repo.go +++ b/pkg/repositories/gormimpl/project_repo.go @@ -10,7 +10,6 @@ import ( "github.com/jinzhu/gorm" - "github.com/lyft/flyteadmin/pkg/common" flyteAdminErrors "github.com/lyft/flyteadmin/pkg/errors" "github.com/lyft/flyteadmin/pkg/repositories/errors" "github.com/lyft/flyteadmin/pkg/repositories/interfaces" @@ -49,13 +48,35 @@ func (r *ProjectRepo) Get(ctx context.Context, projectID string) (models.Project return project, nil } -func (r *ProjectRepo) ListAll(ctx context.Context, sortParameter common.SortParameter) ([]models.Project, error) { +func (r *ProjectRepo) List(ctx context.Context, input interfaces.ListResourceInput) ([]models.Project, error) { var projects []models.Project - var tx = r.db.Where("state != ?", int32(admin.Project_ARCHIVED)) - if sortParameter != nil { - tx = tx.Order(sortParameter.GetGormOrderExpr()) + + tx := r.db.Offset(input.Offset) + if input.Limit != 0 { + tx = tx.Limit(input.Limit) + } + + // Apply filters + // If no filter provided, default to filtering out archived projects + if len(input.InlineFilters) == 0 && len(input.MapFilters) == 0 { + tx = tx.Where("state != ?", int32(admin.Project_ARCHIVED)) + } else { + var err error + tx, err = applyFilters(tx, input.InlineFilters, input.MapFilters) + if err != nil { + return nil, err + } + } + + // Apply sort ordering + if input.SortParameter != nil { + tx = tx.Order(input.SortParameter.GetGormOrderExpr()) } - tx = tx.Find(&projects) + + timer := r.metrics.ListDuration.Start() + tx.Find(&projects) + timer.Stop() + if tx.Error != nil { return nil, r.errorTransformer.ToFlyteAdminError(tx.Error) } diff --git a/pkg/repositories/gormimpl/project_repo_test.go b/pkg/repositories/gormimpl/project_repo_test.go index a17fdd426..951000c0d 100644 --- a/pkg/repositories/gormimpl/project_repo_test.go +++ b/pkg/repositories/gormimpl/project_repo_test.go @@ -7,12 +7,18 @@ import ( mocket "github.com/Selvatico/go-mocket" "github.com/lyft/flyteadmin/pkg/common" "github.com/lyft/flyteadmin/pkg/repositories/errors" + "github.com/lyft/flyteadmin/pkg/repositories/interfaces" "github.com/lyft/flyteadmin/pkg/repositories/models" "github.com/lyft/flyteidl/gen/pb-go/flyteidl/admin" mockScope "github.com/lyft/flytestdlib/promutils" "github.com/stretchr/testify/assert" ) +var alphabeticalSortParam, _ = common.NewSortParameter(admin.Sort{ + Direction: admin.Sort_ASCENDING, + Key: "identifier", +}) + func TestCreateProject(t *testing.T) { projectRepo := NewProjectRepo(GetDbForTest(t), errors.NewTestErrorTransformer(), mockScope.NewTestScope()) GlobalMock := mocket.Catcher.Reset() @@ -57,9 +63,9 @@ func TestGetProject(t *testing.T) { assert.Equal(t, int32(admin.Project_ACTIVE), *output.State) } -func TestListProjects(t *testing.T) { +func testListProjects(input interfaces.ListResourceInput, sql string, t *testing.T) { projectRepo := NewProjectRepo(GetDbForTest(t), errors.NewTestErrorTransformer(), mockScope.NewTestScope()) - projects := make([]map[string]interface{}, 2) + projects := make([]map[string]interface{}, 1) fooProject := make(map[string]interface{}) fooProject["identifier"] = "foo" fooProject["name"] = "foo =)" @@ -67,33 +73,44 @@ func TestListProjects(t *testing.T) { fooProject["state"] = admin.Project_ACTIVE projects[0] = fooProject - barProject := make(map[string]interface{}) - barProject["identifier"] = "bar" - barProject["name"] = "Bar" - barProject["description"] = "Bar description" - barProject["state"] = admin.Project_ACTIVE - projects[1] = barProject - GlobalMock := mocket.Catcher.Reset() GlobalMock.Logging = true - GlobalMock.NewMock().WithQuery(`SELECT * FROM "projects" WHERE "projects"."deleted_at" IS NULL AND ((state != 1)) ORDER BY identifier asc`). + GlobalMock.NewMock().WithQuery(sql). WithReply(projects) - var alphabeticalSortParam, _ = common.NewSortParameter(admin.Sort{ - Direction: admin.Sort_ASCENDING, - Key: "identifier", - }) - output, err := projectRepo.ListAll(context.Background(), alphabeticalSortParam) + output, err := projectRepo.List(context.Background(), input) assert.Nil(t, err) - assert.Len(t, output, 2) + assert.Len(t, output, 1) assert.Equal(t, "foo", output[0].Identifier) assert.Equal(t, "foo =)", output[0].Name) assert.Equal(t, "foo description", output[0].Description) assert.Equal(t, int32(admin.Project_ACTIVE), *output[0].State) - assert.Equal(t, "bar", output[1].Identifier) - assert.Equal(t, "Bar", output[1].Name) - assert.Equal(t, "Bar description", output[1].Description) - assert.Equal(t, int32(admin.Project_ACTIVE), *output[1].State) +} + +func TestListProjects(t *testing.T) { + filter, err := common.NewSingleValueFilter(common.Project, common.Equal, "name", "foo") + assert.Nil(t, err) + testListProjects(interfaces.ListResourceInput{ + Offset: 0, + Limit: 1, + InlineFilters: []common.InlineFilter{filter}, + SortParameter: alphabeticalSortParam, + }, `SELECT * FROM "projects" WHERE "projects"."deleted_at" IS NULL AND ((name = foo)) ORDER BY identifier asc LIMIT 1 OFFSET 0`, t) +} + +func TestListProjects_NoFilters(t *testing.T) { + testListProjects(interfaces.ListResourceInput{ + Offset: 0, + Limit: 1, + SortParameter: alphabeticalSortParam, + }, `SELECT * FROM "projects" WHERE "projects"."deleted_at" IS NULL AND ((state != 1)) ORDER BY identifier asc LIMIT 1 OFFSET 0`, t) +} + +func TestListProjects_NoLimit(t *testing.T) { + testListProjects(interfaces.ListResourceInput{ + Offset: 0, + SortParameter: alphabeticalSortParam, + }, `SELECT * FROM "projects" WHERE "projects"."deleted_at" IS NULL AND ((state != 1)) ORDER BY identifier asc OFFSET 0`, t) } func TestUpdateProject(t *testing.T) { diff --git a/pkg/repositories/interfaces/project_repo.go b/pkg/repositories/interfaces/project_repo.go index cc0c202ba..84eea19be 100644 --- a/pkg/repositories/interfaces/project_repo.go +++ b/pkg/repositories/interfaces/project_repo.go @@ -3,7 +3,6 @@ package interfaces import ( "context" - "github.com/lyft/flyteadmin/pkg/common" "github.com/lyft/flyteadmin/pkg/repositories/models" ) @@ -12,8 +11,8 @@ type ProjectRepoInterface interface { Create(ctx context.Context, project models.Project) error // Returns a matching project when it exists. Get(ctx context.Context, projectID string) (models.Project, error) - // Lists unique projects registered as namespaces - ListAll(ctx context.Context, sortParameter common.SortParameter) ([]models.Project, error) + // Returns projects matching query parameters. + List(ctx context.Context, input ListResourceInput) ([]models.Project, error) // Given a project that exists in the DB and a partial set of fields to update // as a second project (projectUpdate), updates the original project which already // exists in the DB. diff --git a/pkg/repositories/mocks/project_repo.go b/pkg/repositories/mocks/project_repo.go index 1bc892d20..9d56d8138 100644 --- a/pkg/repositories/mocks/project_repo.go +++ b/pkg/repositories/mocks/project_repo.go @@ -3,7 +3,6 @@ package mocks import ( "context" - "github.com/lyft/flyteadmin/pkg/common" "github.com/lyft/flyteadmin/pkg/repositories/interfaces" "github.com/lyft/flyteadmin/pkg/repositories/models" "github.com/lyft/flyteidl/gen/pb-go/flyteidl/admin" @@ -11,7 +10,7 @@ import ( type CreateProjectFunction func(ctx context.Context, project models.Project) error type GetProjectFunction func(ctx context.Context, projectID string) (models.Project, error) -type ListProjectsFunction func(ctx context.Context, sortParameter common.SortParameter) ([]models.Project, error) +type ListProjectsFunction func(ctx context.Context, input interfaces.ListResourceInput) ([]models.Project, error) type UpdateProjectFunction func(ctx context.Context, projectUpdate models.Project) error type MockProjectRepo struct { @@ -39,9 +38,9 @@ func (r *MockProjectRepo) Get(ctx context.Context, projectID string) (models.Pro }, nil } -func (r *MockProjectRepo) ListAll(ctx context.Context, sortParameter common.SortParameter) ([]models.Project, error) { +func (r *MockProjectRepo) List(ctx context.Context, input interfaces.ListResourceInput) ([]models.Project, error) { if r.ListProjectsFunction != nil { - return r.ListProjectsFunction(ctx, sortParameter) + return r.ListProjectsFunction(ctx, input) } return make([]models.Project, 0), nil } From aa420b10a34ea6edf9f94cf7c7942173653bb0b8 Mon Sep 17 00:00:00 2001 From: Honnix Date: Wed, 2 Dec 2020 08:50:02 +0100 Subject: [PATCH 12/22] Be more generous validating update project (#145) * Be more generous validating update project --- pkg/manager/impl/project_manager_test.go | 9 +++++---- pkg/manager/impl/validation/project_validator.go | 9 +++++---- pkg/manager/impl/validation/project_validator_test.go | 8 +------- 3 files changed, 11 insertions(+), 15 deletions(-) diff --git a/pkg/manager/impl/project_manager_test.go b/pkg/manager/impl/project_manager_test.go index c25307f81..2d9200fd6 100644 --- a/pkg/manager/impl/project_manager_test.go +++ b/pkg/manager/impl/project_manager_test.go @@ -6,7 +6,6 @@ import ( "testing" "github.com/lyft/flyteadmin/pkg/common" - "github.com/lyft/flyteadmin/pkg/manager/impl/shared" "github.com/lyft/flyteadmin/pkg/manager/impl/testutils" "github.com/lyft/flyteadmin/pkg/repositories/interfaces" @@ -209,6 +208,7 @@ func TestProjectManager_UpdateProject(t *testing.T) { assert.Equal(t, "project-id", projectUpdate.Identifier) assert.Equal(t, "new-project-name", projectUpdate.Name) assert.Equal(t, "new-project-description", projectUpdate.Description) + assert.Equal(t, int32(admin.Project_ACTIVE), *projectUpdate.State) return nil } projectManager := NewProjectManager(mockRepository, @@ -218,6 +218,7 @@ func TestProjectManager_UpdateProject(t *testing.T) { Id: "project-id", Name: "new-project-name", Description: "new-project-description", + State: admin.Project_ACTIVE, }) assert.Nil(t, err) assert.True(t, updateFuncCalled) @@ -260,8 +261,8 @@ func TestProjectManager_UpdateProject_ErrorDueToInvalidProjectName(t *testing.T) runtimeMocks.NewMockConfigurationProvider( getMockApplicationConfigForProjectManagerTest(), nil, nil, nil, nil, nil)) _, err := projectManager.UpdateProject(context.Background(), admin.Project{ - Id: "project-id", - // No project name + Id: "project-id", + Name: "longnamelongnamelongnamelongnamelongnamelongnamelongnamelongnamel", }) - assert.Equal(t, shared.GetMissingArgumentError("project_name"), err) + assert.EqualError(t, err, "project_name cannot exceed 64 characters") } diff --git a/pkg/manager/impl/validation/project_validator.go b/pkg/manager/impl/validation/project_validator.go index 212113240..5cd60934c 100644 --- a/pkg/manager/impl/validation/project_validator.go +++ b/pkg/manager/impl/validation/project_validator.go @@ -25,7 +25,11 @@ func ValidateProjectRegisterRequest(request admin.ProjectRegisterRequest) error if request.Project == nil { return shared.GetMissingArgumentError(shared.Project) } - return ValidateProject(*request.Project) + project := *request.Project + if err := ValidateEmptyStringField(project.Name, projectName); err != nil { + return err + } + return ValidateProject(project) } func ValidateProject(project admin.Project) error { @@ -38,9 +42,6 @@ func ValidateProject(project admin.Project) error { if errs := validation.IsDNS1123Label(project.Id); len(errs) > 0 { return errors.NewFlyteAdminErrorf(codes.InvalidArgument, "invalid project id [%s]: %v", project.Id, errs) } - if err := ValidateEmptyStringField(project.Name, projectName); err != nil { - return err - } if err := ValidateMaxLengthStringField(project.Name, projectName, maxNameLength); err != nil { return err } diff --git a/pkg/manager/impl/validation/project_validator_test.go b/pkg/manager/impl/validation/project_validator_test.go index 3be5933a7..e6c00a258 100644 --- a/pkg/manager/impl/validation/project_validator_test.go +++ b/pkg/manager/impl/validation/project_validator_test.go @@ -145,8 +145,8 @@ func TestValidateProjectRegisterRequest(t *testing.T) { func TestValidateProject_ValidProject(t *testing.T) { assert.Nil(t, ValidateProject(admin.Project{ Id: "proj", - Name: "proj", Description: "An amazing description for this project", + State: admin.Project_ARCHIVED, Labels: &admin.Labels{ Values: map[string]string{ "foo": "bar", @@ -182,12 +182,6 @@ func TestValidateProject(t *testing.T) { "characters or '-', and must start and end with an alphanumeric character (e.g. 'my-name', or " + "'123-abc', regex used for validation is '[a-z0-9]([-a-z0-9]*[a-z0-9])?')]", }, - { - project: admin.Project{ - Id: "proj", - }, - expectedError: "missing project_name", - }, { project: admin.Project{ Id: "proj", From 5274cbfd66c65bd8210e52b68e65737ce309349c Mon Sep 17 00:00:00 2001 From: Yee Hing Tong Date: Tue, 5 Jan 2021 15:10:49 -0800 Subject: [PATCH 13/22] go get propeller at v0.5.12 (#146) --- go.mod | 6 +++++- go.sum | 5 +++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 129310191..2b8df513a 100644 --- a/go.mod +++ b/go.mod @@ -5,11 +5,13 @@ go 1.13 require ( cloud.google.com/go v0.56.0 cloud.google.com/go/storage v1.6.0 + github.com/GoogleCloudPlatform/spark-on-k8s-operator v0.1.3 // indirect github.com/NYTimes/gizmo v1.3.5 github.com/Selvatico/go-mocket v1.0.7 github.com/aws/aws-sdk-go v1.29.23 github.com/benbjohnson/clock v1.0.0 github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b // indirect + github.com/coreos/etcd v3.3.15+incompatible // indirect github.com/coreos/go-oidc v2.2.1+incompatible github.com/evanphx/json-patch v4.5.0+incompatible github.com/gogo/protobuf v1.3.1 @@ -26,9 +28,10 @@ require ( github.com/kelseyhightower/envconfig v1.4.0 // indirect github.com/lib/pq v1.3.0 github.com/lyft/flyteidl v0.18.11 - github.com/lyft/flytepropeller v0.3.17 + github.com/lyft/flytepropeller v0.5.12 github.com/lyft/flytestdlib v0.3.9 github.com/magiconair/properties v1.8.1 + github.com/mitchellh/go-ps v1.0.0 // indirect github.com/mitchellh/mapstructure v1.1.2 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.5.0 @@ -45,6 +48,7 @@ require ( k8s.io/apimachinery v0.17.3 k8s.io/client-go v11.0.1-0.20190918222721-c0e3722d5cf0+incompatible sigs.k8s.io/controller-runtime v0.5.1 + sigs.k8s.io/testing_frameworks v0.1.2 // indirect ) // Pin the version of client-go to something that's compatible with katrogan's fork of api and apimachinery diff --git a/go.sum b/go.sum index 3d1e0d901..c98e4d2d3 100644 --- a/go.sum +++ b/go.sum @@ -464,11 +464,15 @@ github.com/lyft/flyteidl v0.17.0/go.mod h1:/zQXxuHO11u/saxTTZc8oYExIGEShXB+xCB1/ github.com/lyft/flyteidl v0.18.0/go.mod h1:/zQXxuHO11u/saxTTZc8oYExIGEShXB+xCB1/F1Cu20= github.com/lyft/flyteidl v0.18.6 h1:HGbxHI8avEDvoPqcO2+/BoJVcP9sjOj4qwJ/wNRWuoA= github.com/lyft/flyteidl v0.18.6/go.mod h1:/zQXxuHO11u/saxTTZc8oYExIGEShXB+xCB1/F1Cu20= +github.com/lyft/flyteidl v0.18.9/go.mod h1:/zQXxuHO11u/saxTTZc8oYExIGEShXB+xCB1/F1Cu20= github.com/lyft/flyteidl v0.18.11 h1:24NaFYWxANhRbwKfvkgu8axGTWUcl1tgZBqNJutKNJ8= github.com/lyft/flyteidl v0.18.11/go.mod h1:/zQXxuHO11u/saxTTZc8oYExIGEShXB+xCB1/F1Cu20= github.com/lyft/flyteplugins v0.5.1/go.mod h1:8zhqFG9BzbHNQGEXzGYltTJLD+KTmQZkanxXgeFI25c= +github.com/lyft/flyteplugins v0.5.20/go.mod h1:1G2YORr5JZhfl4dyHGIO4/+ShNMul5DPnoBEdpJfuNc= github.com/lyft/flytepropeller v0.3.17 h1:a2PVqWjnn8oNEeayAqNizMAtEixl/F3S4vd8z4kbiqI= github.com/lyft/flytepropeller v0.3.17/go.mod h1:T8Utxqv7B5USAX9c/Qh0lBbKXHFSgOwwaISOd9h36P4= +github.com/lyft/flytepropeller v0.5.12 h1:TD390M6iyQQznlu5cuThModN0rN9N0GwSbn5eXIWgns= +github.com/lyft/flytepropeller v0.5.12/go.mod h1:EEMFU+W+VZ1eSMZQqjFgFUFgWTxQmgaXqqiNDsfMc8I= github.com/lyft/flytestdlib v0.3.0 h1:nIkX4MlyYdcLLzaF35RI2P5BhARt+qMgHoFto8eVNzU= github.com/lyft/flytestdlib v0.3.0/go.mod h1:LJPPJlkFj+wwVWMrQT3K5JZgNhZi2mULsCG4ZYhinhU= github.com/lyft/flytestdlib v0.3.9 h1:NaKp9xkeWWwhVvqTOcR/FqlASy1N2gu/kN7PVe4S7YI= @@ -612,6 +616,7 @@ github.com/prometheus/procfs v0.0.10/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+G github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= +github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= From 26785a4814650e0a09153de9c1ba64e676c312fb Mon Sep 17 00:00:00 2001 From: Katrina Rogan Date: Wed, 6 Jan 2021 14:00:38 -0800 Subject: [PATCH 14/22] Update to using latest flytepropeller v0.5.13 (#148) --- go.mod | 6 +----- go.sum | 39 ++------------------------------------- 2 files changed, 3 insertions(+), 42 deletions(-) diff --git a/go.mod b/go.mod index 2b8df513a..b774186c6 100644 --- a/go.mod +++ b/go.mod @@ -5,13 +5,11 @@ go 1.13 require ( cloud.google.com/go v0.56.0 cloud.google.com/go/storage v1.6.0 - github.com/GoogleCloudPlatform/spark-on-k8s-operator v0.1.3 // indirect github.com/NYTimes/gizmo v1.3.5 github.com/Selvatico/go-mocket v1.0.7 github.com/aws/aws-sdk-go v1.29.23 github.com/benbjohnson/clock v1.0.0 github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b // indirect - github.com/coreos/etcd v3.3.15+incompatible // indirect github.com/coreos/go-oidc v2.2.1+incompatible github.com/evanphx/json-patch v4.5.0+incompatible github.com/gogo/protobuf v1.3.1 @@ -28,10 +26,9 @@ require ( github.com/kelseyhightower/envconfig v1.4.0 // indirect github.com/lib/pq v1.3.0 github.com/lyft/flyteidl v0.18.11 - github.com/lyft/flytepropeller v0.5.12 + github.com/lyft/flytepropeller v0.5.13 github.com/lyft/flytestdlib v0.3.9 github.com/magiconair/properties v1.8.1 - github.com/mitchellh/go-ps v1.0.0 // indirect github.com/mitchellh/mapstructure v1.1.2 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.5.0 @@ -48,7 +45,6 @@ require ( k8s.io/apimachinery v0.17.3 k8s.io/client-go v11.0.1-0.20190918222721-c0e3722d5cf0+incompatible sigs.k8s.io/controller-runtime v0.5.1 - sigs.k8s.io/testing_frameworks v0.1.2 // indirect ) // Pin the version of client-go to something that's compatible with katrogan's fork of api and apimachinery diff --git a/go.sum b/go.sum index c98e4d2d3..a2b8e4f76 100644 --- a/go.sum +++ b/go.sum @@ -36,7 +36,6 @@ dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7 github.com/Azure/azure-sdk-for-go v32.5.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go v38.2.0+incompatible h1:ZeCdp1E/V5lI8oLR/BjWQh0OW9aFBYlgXGKRVIWNPXY= github.com/Azure/azure-sdk-for-go v38.2.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-sdk-for-go v39.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go v40.3.0+incompatible h1:NthZg3psrLxvQLN6rVm07pZ9mv2wvGNaBNGQ3fnPvLE= github.com/Azure/azure-sdk-for-go v40.3.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= @@ -109,7 +108,6 @@ github.com/aws/aws-sdk-go v1.19.18/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpi github.com/aws/aws-sdk-go v1.23.4/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.28.9 h1:grIuBQc+p3dTRXerh5+2OxSuWFi0iXuxbFdTSg0jaW0= github.com/aws/aws-sdk-go v1.28.9/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go v1.28.11/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.29.23 h1:wtiGLOzxAP755OfuVTDIy/NbUIYEDxbIbBEDfNhUpeU= github.com/aws/aws-sdk-go v1.29.23/go.mod h1:1KvfttTE3SPKMpo8g2c6jL3ZKfXtFvKscTgahTma5Xg= github.com/aws/aws-sdk-go-v2 v0.20.0/go.mod h1:2LhT7UgHOXK3UXONKI5OMgIyoQL6zTAw/jwIeX6yqzw= @@ -142,10 +140,8 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/coocood/freecache v1.1.0 h1:ENiHOsWdj1BrrlPwblhbn4GdAsMymK3pZORJ+bJGAjA= github.com/coocood/freecache v1.1.0/go.mod h1:ePwxCDzOYvARfHdr1pByNct1at3CoKnsipOHwKlNbzI= -github.com/coreos/bbolt v1.3.1-coreos.6/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/etcd v3.3.15+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-oidc v2.1.0+incompatible h1:sdJrfw8akMnCuUlaZU3tE/uYXFgfqom8DBE9so9EBsM= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= @@ -298,7 +294,6 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.0.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= @@ -362,7 +357,6 @@ github.com/graymeta/stow v0.2.4/go.mod h1:+0vRL9oMECKjPMP7OeVWl8EIqRCpFwDlth3mrA github.com/graymeta/stow v0.2.5 h1:YFSo4nsAU4Fbi4r/neLIgVYlrMzA1ReDUkdLYTQm/RM= github.com/graymeta/stow v0.2.5/go.mod h1:+0vRL9oMECKjPMP7OeVWl8EIqRCpFwDlth3mrAeV2Kw= github.com/gregjones/httpcache v0.0.0-20170728041850-787624de3eb7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/grpc-ecosystem/go-grpc-middleware v0.0.0-20190222133341-cfaf5686ec79/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 h1:THDBEeQ9xZ8JEaCLyLQqXMMdRqNr0QAUJTIkQAUtFjg= @@ -371,7 +365,6 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.2.0 h1:0IKlLyQ3Hs9nDaiK5cSHAGmcQ github.com/grpc-ecosystem/go-grpc-middleware v1.2.0/go.mod h1:mJzapYve32yjrKlk9GbyCZHuPgZsrbyIbyKhSzOpg6s= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.3.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.12.2 h1:D0EVSTwQoQOyfY35QNSuPJA4jpZRtkoGYWQMB7XNg5o= @@ -461,18 +454,12 @@ github.com/lyft/apimachinery v0.0.0-20191031200210-047e3ea32d7f h1:PGuAMDzAen0Au github.com/lyft/apimachinery v0.0.0-20191031200210-047e3ea32d7f/go.mod h1:llRdnznGEAqC3DcNm6yEj472xaFVfLM7hnYofMb12tQ= github.com/lyft/datacatalog v0.2.1/go.mod h1:ktrPvzTDUwHO5Lv0hLH38zLHnOJ++rGoAO0iQ/sIPJ4= github.com/lyft/flyteidl v0.17.0/go.mod h1:/zQXxuHO11u/saxTTZc8oYExIGEShXB+xCB1/F1Cu20= -github.com/lyft/flyteidl v0.18.0/go.mod h1:/zQXxuHO11u/saxTTZc8oYExIGEShXB+xCB1/F1Cu20= -github.com/lyft/flyteidl v0.18.6 h1:HGbxHI8avEDvoPqcO2+/BoJVcP9sjOj4qwJ/wNRWuoA= -github.com/lyft/flyteidl v0.18.6/go.mod h1:/zQXxuHO11u/saxTTZc8oYExIGEShXB+xCB1/F1Cu20= github.com/lyft/flyteidl v0.18.9/go.mod h1:/zQXxuHO11u/saxTTZc8oYExIGEShXB+xCB1/F1Cu20= github.com/lyft/flyteidl v0.18.11 h1:24NaFYWxANhRbwKfvkgu8axGTWUcl1tgZBqNJutKNJ8= github.com/lyft/flyteidl v0.18.11/go.mod h1:/zQXxuHO11u/saxTTZc8oYExIGEShXB+xCB1/F1Cu20= -github.com/lyft/flyteplugins v0.5.1/go.mod h1:8zhqFG9BzbHNQGEXzGYltTJLD+KTmQZkanxXgeFI25c= github.com/lyft/flyteplugins v0.5.20/go.mod h1:1G2YORr5JZhfl4dyHGIO4/+ShNMul5DPnoBEdpJfuNc= -github.com/lyft/flytepropeller v0.3.17 h1:a2PVqWjnn8oNEeayAqNizMAtEixl/F3S4vd8z4kbiqI= -github.com/lyft/flytepropeller v0.3.17/go.mod h1:T8Utxqv7B5USAX9c/Qh0lBbKXHFSgOwwaISOd9h36P4= -github.com/lyft/flytepropeller v0.5.12 h1:TD390M6iyQQznlu5cuThModN0rN9N0GwSbn5eXIWgns= -github.com/lyft/flytepropeller v0.5.12/go.mod h1:EEMFU+W+VZ1eSMZQqjFgFUFgWTxQmgaXqqiNDsfMc8I= +github.com/lyft/flytepropeller v0.5.13 h1:GcEmMgE5BFmjyfo1U9nqseQEKHfHsT0FT1IXfhC3/P4= +github.com/lyft/flytepropeller v0.5.13/go.mod h1:P63eiQpUI/57ifXNoHc/NSszU8PYLcJ+fujI4wQ99rI= github.com/lyft/flytestdlib v0.3.0 h1:nIkX4MlyYdcLLzaF35RI2P5BhARt+qMgHoFto8eVNzU= github.com/lyft/flytestdlib v0.3.0/go.mod h1:LJPPJlkFj+wwVWMrQT3K5JZgNhZi2mULsCG4ZYhinhU= github.com/lyft/flytestdlib v0.3.9 h1:NaKp9xkeWWwhVvqTOcR/FqlASy1N2gu/kN7PVe4S7YI= @@ -507,7 +494,6 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0j github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -531,7 +517,6 @@ github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d/go.mod h1:YUTz3bUH github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.4.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w= @@ -540,7 +525,6 @@ github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+ github.com/onsi/ginkgo v1.11.0 h1:JAKSXpt1YjtLA7YpPiqO9ss6sNXEsPfSGdwN0UHqzrw= github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= -github.com/onsi/gomega v1.3.0/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo= @@ -581,7 +565,6 @@ github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.3.0 h1:miYCvYqFXtl/J9FIy8eNpBfYthAEFg+Ys0XyUVEcDsc= github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= -github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_golang v1.5.0 h1:Ctq0iGpCmr3jeP77kbF2UxgvRwzWWz+4Bh9/vJTyg1A= github.com/prometheus/client_golang v1.5.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= @@ -616,7 +599,6 @@ github.com/prometheus/procfs v0.0.10/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+G github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= -github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -636,7 +618,6 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykE github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/soheilhy/cmux v0.1.3/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= @@ -691,7 +672,6 @@ github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijb github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= -github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -709,14 +689,11 @@ go.opencensus.io v0.22.2 h1:75k/FF0Q2YM8QYo07VPddOLBslDt1MZOdEslOHvmzAs= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.uber.org/atomic v0.0.0-20181018215023-8dc6146f7569/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/multierr v0.0.0-20180122172545-ddea229ff1df/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/zap v0.0.0-20180814183419-67bc79d13d15/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= @@ -780,7 +757,6 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180112015858-5ccada7d0a7b/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -839,7 +815,6 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 h1:qwRHBd0NqMbJxfbotnDhm2ByMI1Shq4Y6oRJo21SGJA= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180117170059-2c42eef0765b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -888,7 +863,6 @@ golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20171227012246-e19ae1496984/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -1056,7 +1030,6 @@ gopkg.in/square/go-jose.v2 v2.4.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= -gopkg.in/yaml.v2 v2.0.0/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -1079,17 +1052,12 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt honnef.co/go/tools v0.0.1-2020.1.3 h1:sXmLre5bzIR6ypkjXCDI3jHPssRhc8KD/Ome589sc3U= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= k8s.io/apiextensions-apiserver v0.0.0-20190409022649-727a075fdec8/go.mod h1:IxkesAMoaCRoLrPJdZNZUQp9NfZnzqaVzLhb2VEQzXE= -k8s.io/apiextensions-apiserver v0.0.0-20190918161926-8f644eb6e783 h1:V6ndwCPoao1yZ52agqOKaUAl7DYWVGiXjV7ePA2i610= -k8s.io/apiextensions-apiserver v0.0.0-20190918161926-8f644eb6e783/go.mod h1:xvae1SZB3E17UpV59AWc271W/Ph25N+bjPyR63X6tPY= k8s.io/apiextensions-apiserver v0.17.2 h1:cP579D2hSZNuO/rZj9XFRzwJNYb41DbNANJb6Kolpss= k8s.io/apiextensions-apiserver v0.17.2/go.mod h1:4KdMpjkEjjDI2pPfBA15OscyNldHWdBCfsWMDWAmSTs= -k8s.io/apiserver v0.0.0-20190918160949-bfa5e2e684ad/go.mod h1:XPCXEwhjaFN29a8NldXA901ElnKeKLrLtREO9ZhFyhg= k8s.io/apiserver v0.17.2/go.mod h1:lBmw/TtQdtxvrTk0e2cgtOxHizXI+d0mmGQURIHQZlo= k8s.io/client-go v0.0.0-20191016111102-bec269661e48 h1:C2XVy2z0dV94q9hSSoCuTPp1KOG7IegvbdXuz9VGxoU= k8s.io/client-go v0.0.0-20191016111102-bec269661e48/go.mod h1:hrwktSwYGI4JK+TJA3dMaFyyvHVi/aLarVHpbs8bgCU= -k8s.io/code-generator v0.0.0-20190912054826-cd179ad6a269/go.mod h1:V5BD6M4CyaN5m+VthcclXWsVcT1Hu+glwa1bi3MIsyE= k8s.io/code-generator v0.17.2/go.mod h1:DVmfPQgxQENqDIzVR2ddLXMH34qeszkKSdH/N+s+38s= -k8s.io/component-base v0.0.0-20190918160511-547f6c5d7090/go.mod h1:933PBGtQFJky3TEwYx4aEPZ4IxqhWh3R6DCmzqIn1hA= k8s.io/component-base v0.17.2/go.mod h1:zMPW3g5aH7cHJpKYQ/ZsGMcgbsA/VyhEugF3QT1awLs= k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20190822140433-26a664648505/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= @@ -1120,15 +1088,12 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/controller-runtime v0.2.0/go.mod h1:ZHqrRDZi3f6BzONcvlUxkqCKgwasGk5FZrnSv9TVZF4= -sigs.k8s.io/controller-runtime v0.4.0/go.mod h1:ApC79lpY3PHW9xj/w9pj+lYkLgwAAUZwfXkME1Lajns= sigs.k8s.io/controller-runtime v0.5.1 h1:TNidCfVoU/cs2i+9xoTcL/l7yhl0bDhYXU0NCG6wmiE= sigs.k8s.io/controller-runtime v0.5.1/go.mod h1:Uojny7gvg55YLQnEGnPzRE3dC4ik2tRlZJgOUCWXAV4= sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= -sigs.k8s.io/structured-merge-diff v0.0.0-20190817042607-6149e4549fca/go.mod h1:IIgPezJWb76P0hotTxzDbWsMYB8APh18qZnxkomBpxA= sigs.k8s.io/structured-merge-diff v1.0.1-0.20191108220359-b1b620dd3f06/go.mod h1:/ULNhyfzRopfcjskuui0cTITekDduZ7ycKN3oUT9R18= sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= sigs.k8s.io/testing_frameworks v0.1.1/go.mod h1:VVBKrHmJ6Ekkfz284YKhQePcdycOzNH9qL6ht1zEr/U= -sigs.k8s.io/testing_frameworks v0.1.2/go.mod h1:ToQrwSC3s8Xf/lADdZp3Mktcql9CG0UAmdJG9th5i0w= sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= From 44f525f49d846906b38e48abd446511a7e2049e1 Mon Sep 17 00:00:00 2001 From: Haytham AbuelFutuh Date: Mon, 18 Jan 2021 17:10:33 -0800 Subject: [PATCH 15/22] Update propeller to 0.5.14 (#149) * Update propeller * cleanup --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index b774186c6..da522d0db 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,7 @@ require ( github.com/kelseyhightower/envconfig v1.4.0 // indirect github.com/lib/pq v1.3.0 github.com/lyft/flyteidl v0.18.11 - github.com/lyft/flytepropeller v0.5.13 + github.com/lyft/flytepropeller v0.5.14 github.com/lyft/flytestdlib v0.3.9 github.com/magiconair/properties v1.8.1 github.com/mitchellh/mapstructure v1.1.2 diff --git a/go.sum b/go.sum index a2b8e4f76..6eb554af8 100644 --- a/go.sum +++ b/go.sum @@ -458,8 +458,8 @@ github.com/lyft/flyteidl v0.18.9/go.mod h1:/zQXxuHO11u/saxTTZc8oYExIGEShXB+xCB1/ github.com/lyft/flyteidl v0.18.11 h1:24NaFYWxANhRbwKfvkgu8axGTWUcl1tgZBqNJutKNJ8= github.com/lyft/flyteidl v0.18.11/go.mod h1:/zQXxuHO11u/saxTTZc8oYExIGEShXB+xCB1/F1Cu20= github.com/lyft/flyteplugins v0.5.20/go.mod h1:1G2YORr5JZhfl4dyHGIO4/+ShNMul5DPnoBEdpJfuNc= -github.com/lyft/flytepropeller v0.5.13 h1:GcEmMgE5BFmjyfo1U9nqseQEKHfHsT0FT1IXfhC3/P4= -github.com/lyft/flytepropeller v0.5.13/go.mod h1:P63eiQpUI/57ifXNoHc/NSszU8PYLcJ+fujI4wQ99rI= +github.com/lyft/flytepropeller v0.5.14 h1:6S93Q1tTnoc5iFKe4Sw4qZzQHZgfuTQd6zUdxaZGAik= +github.com/lyft/flytepropeller v0.5.14/go.mod h1:P63eiQpUI/57ifXNoHc/NSszU8PYLcJ+fujI4wQ99rI= github.com/lyft/flytestdlib v0.3.0 h1:nIkX4MlyYdcLLzaF35RI2P5BhARt+qMgHoFto8eVNzU= github.com/lyft/flytestdlib v0.3.0/go.mod h1:LJPPJlkFj+wwVWMrQT3K5JZgNhZi2mULsCG4ZYhinhU= github.com/lyft/flytestdlib v0.3.9 h1:NaKp9xkeWWwhVvqTOcR/FqlASy1N2gu/kN7PVe4S7YI= From 63e0c279c4f4d2742f0819f2456c6aa4116fd3db Mon Sep 17 00:00:00 2001 From: Katrina Rogan Date: Fri, 22 Jan 2021 16:21:32 -0800 Subject: [PATCH 16/22] Filter executions by user (#150) --- go.mod | 2 +- go.sum | 14 ++++++++++++++ pkg/repositories/config/migrations.go | 9 +++++++++ pkg/repositories/models/execution.go | 3 +++ pkg/repositories/transformers/execution.go | 1 + pkg/repositories/transformers/execution_test.go | 1 + 6 files changed, 29 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index da522d0db..8f0419ca4 100644 --- a/go.mod +++ b/go.mod @@ -36,7 +36,7 @@ require ( github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.6.1 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d - golang.org/x/tools v0.0.0-20200818005847-188abfa75333 // indirect + golang.org/x/tools v0.1.0 // indirect google.golang.org/api v0.20.0 google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940 google.golang.org/grpc v1.28.0 diff --git a/go.sum b/go.sum index 6eb554af8..4ee515ec9 100644 --- a/go.sum +++ b/go.sum @@ -676,6 +676,7 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= @@ -796,6 +797,8 @@ golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7 golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344 h1:vGXIOMxbNfDTk/aXCmfdLgkrSV+Z2tcbze+pEc3v5W4= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -814,6 +817,8 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 h1:qwRHBd0NqMbJxfbotnDhm2ByMI1Shq4Y6oRJo21SGJA= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -860,12 +865,17 @@ golang.org/x/sys v0.0.0-20200327173247-9dae0f8f5775 h1:TC0v2RSO1u2kn1ZugjrFXkRZA golang.org/x/sys v0.0.0-20200327173247-9dae0f8f5775/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200331124033-c3d80250170d h1:nc5K6ox/4lTFbMVSL9WRR81ixkcwXThoiF6yf+R9scA= golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 h1:myAQVi0cGEoqQVR5POX+8RR2mrocKqNN1hmeMqhX27k= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -916,10 +926,14 @@ golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjs golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200818005847-188abfa75333 h1:a6ryybeZHQf5qnBc6IwRfVnI/75UmdtJo71f0//8Dqo= golang.org/x/tools v0.0.0-20200818005847-188abfa75333/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.0.1/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3mwe7XcUU= gomodules.xyz/jsonpatch/v2 v2.1.0 h1:Phva6wqu+xR//Njw6iorylFFgn/z547tw5Ne3HZPQ+k= gomodules.xyz/jsonpatch/v2 v2.1.0/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3mwe7XcUU= diff --git a/pkg/repositories/config/migrations.go b/pkg/repositories/config/migrations.go index 9ba6640d2..18b739692 100644 --- a/pkg/repositories/config/migrations.go +++ b/pkg/repositories/config/migrations.go @@ -287,4 +287,13 @@ var Migrations = []*gormigrate.Migration{ return tx.Exec("UPDATE projects set state = NULL").Error }, }, + { + ID: "2021-01-22-execution-user", + Migrate: func(tx *gorm.DB) error { + return tx.AutoMigrate(&models.Execution{}).Error + }, + Rollback: func(tx *gorm.DB) error { + return tx.Model(&models.Execution{}).DropColumn("user").Error + }, + }, } diff --git a/pkg/repositories/models/execution.go b/pkg/repositories/models/execution.go index 3bfea3df4..5ce0d69b0 100644 --- a/pkg/repositories/models/execution.go +++ b/pkg/repositories/models/execution.go @@ -56,4 +56,7 @@ type Execution struct { ErrorKind *string `gorm:"index"` // Execution Error Code nullable ErrorCode *string + // The user responsible for launching this execution. + // This is also stored in the spec but promoted as a column for filtering. + User string `gorm:"index"` } diff --git a/pkg/repositories/transformers/execution.go b/pkg/repositories/transformers/execution.go index 3d9481011..74063fcc4 100644 --- a/pkg/repositories/transformers/execution.go +++ b/pkg/repositories/transformers/execution.go @@ -90,6 +90,7 @@ func CreateExecutionModel(input CreateExecutionModelInput) (*models.Execution, e Cluster: input.Cluster, InputsURI: input.InputsURI, UserInputsURI: input.UserInputsURI, + User: requestSpec.Metadata.Principal, } // A reference launch entity can be one of either or a task OR launch plan. Traditionally, workflows are executed // with a reference launch plan which is why this behavior is the default below. diff --git a/pkg/repositories/transformers/execution_test.go b/pkg/repositories/transformers/execution_test.go index 6c82dda6d..861b320da 100644 --- a/pkg/repositories/transformers/execution_test.go +++ b/pkg/repositories/transformers/execution_test.go @@ -90,6 +90,7 @@ func TestCreateExecutionModel(t *testing.T) { } expectedSpecBytes, _ := proto.Marshal(expectedSpec) assert.Equal(t, expectedSpecBytes, execution.Spec) + assert.Equal(t, execution.User, principal) expectedCreatedAt, _ := ptypes.TimestampProto(createdAt) expectedClosure, _ := proto.Marshal(&admin.ExecutionClosure{ From f577697419b4088829301b8af62496f7b2e89e04 Mon Sep 17 00:00:00 2001 From: Haytham Abuelfutuh Date: Wed, 3 Feb 2021 13:49:31 -0800 Subject: [PATCH 17/22] Update CI post migration --- .github/workflows/master.yml | 24 +++--------------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 49e25a6dc..e4c717764 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -94,7 +94,7 @@ jobs: version: latest args: release --rm-dist env: - GITHUB_TOKEN: ${{ secrets.GORELEASER_TOKEN }} + GITHUB_TOKEN: ${{ secrets.FLYTE_BOT_PAT }} push-github: name: Push to Github Registry @@ -107,32 +107,14 @@ jobs: - name: Push Docker Image to Github Registry uses: whoan/docker-build-with-cache-action@v5 with: - username: "${{ secrets.PUBLISH_CONTAINER_ACTOR }}" - password: "${{ secrets.PUBLISH_CONTAINER }}" + username: "${{ secrets.FLYTE_BOT_USERNAME }}" + password: "${{ secrets.FLYTE_BOT_PAT }}" image_name: ${{ github.repository }} image_tag: latest,${{ github.sha }},${{ needs.bump-version.outputs.version }} push_git_tag: true registry: ghcr.io build_extra_args: "--compress=true" - push-dockerhub: - name: Push to Dockerhub - runs-on: ubuntu-latest - needs: bump-version - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: '0' - - name: Push Docker Image to Dockerhub - uses: whoan/docker-build-with-cache-action@v5 - with: - username: "${{ secrets.DOCKERHUB_USERNAME }}" - password: "${{ secrets.DOCKERHUB_PASSWORD }}" - image_name: ${{ secrets.DOCKERHUB_IMAGE_NAME }} - image_tag: latest,${{ github.sha }},${{ needs.bump-version.outputs.version }} - push_git_tag: true - build_extra_args: "--compress=true" - tests-lint: name: Run tests and lint runs-on: ubuntu-latest From 955dbefcdf4b7b24bc16b4e6882f61940ae0345f Mon Sep 17 00:00:00 2001 From: Haytham Abuelfutuh Date: Mon, 8 Feb 2021 11:48:12 -0800 Subject: [PATCH 18/22] Update codecov link --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 4f4fdd762..b7b2a55a7 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,17 @@ Flyteadmin ============= -[![Current Release](https://img.shields.io/github/release/lyft/flyteadmin.svg)](https://github.com/lyft/flyteadmin/releases/latest) -![Master](https://github.com/lyft/flyteadmin/workflows/Master/badge.svg) +[![Current Release](https://img.shields.io/github/release/flyteorg/flyteadmin.svg)](https://github.com/flyteorg/flyteadmin/releases/latest) +![Master](https://github.com/flyteorg/flyteadmin/workflows/Master/badge.svg) [![GoDoc](https://godoc.org/github.com/lyft/flyteadmin?status.svg)](https://pkg.go.dev/mod/github.com/lyft/flyteadmin) [![License](https://img.shields.io/badge/LICENSE-Apache2.0-ff69b4.svg)](http://www.apache.org/licenses/LICENSE-2.0.html) -[![CodeCoverage](https://img.shields.io/codecov/c/github/lyft/flyteadmin.svg)](https://codecov.io/gh/lyft/flyteadmin) +[![CodeCoverage](https://img.shields.io/codecov/c/github/flyteorg/flyteadmin.svg)](https://codecov.io/gh/flyteorg/flyteadmin) [![Go Report Card](https://goreportcard.com/badge/github.com/lyft/flyteadmin)](https://goreportcard.com/report/github.com/lyft/flyteadmin) -![Commit activity](https://img.shields.io/github/commit-activity/w/lyft/flyteadmin.svg?style=plastic) -![Commit since last release](https://img.shields.io/github/commits-since/lyft/flyteadmin/latest.svg?style=plastic) +![Commit activity](https://img.shields.io/github/commit-activity/w/flyteorg/flyteadmin.svg?style=plastic) +![Commit since last release](https://img.shields.io/github/commits-since/flyteorg/flyteadmin/latest.svg?style=plastic) Flyteadmin is the control plane for Flyte responsible for managing entities (task, workflows, launch plans) and administering workflow executions. Flyteadmin implements the -[AdminService](https://github.com/lyft/flyteidl/blob/master/protos/flyteidl/service/admin.proto) which +[AdminService](https://github.com/flyteorg/flyteidl/blob/master/protos/flyteidl/service/admin.proto) which defines a stateless REST/gRPC service for interacting with registered Flyte entities and executions. Flyteadmin uses a relational style Metadata Store abstracted by [GORM](http://gorm.io/) ORM library. From 659635fd04f5a51c6ae4ded47fe2321a53be39db Mon Sep 17 00:00:00 2001 From: tnsetting <38207208+tnsetting@users.noreply.github.com> Date: Tue, 9 Feb 2021 19:15:15 +0100 Subject: [PATCH 19/22] Publish raw events (#151) --- flyteadmin_config.yaml | 8 + pkg/async/notifications/factory.go | 50 +++++ .../implementations/event_publisher.go | 105 ++++++++++ .../implementations/event_publisher_test.go | 173 ++++++++++++++++ pkg/manager/impl/execution_manager.go | 21 +- pkg/manager/impl/execution_manager_test.go | 194 +++++------------- pkg/manager/impl/node_execution_manager.go | 37 ++-- .../impl/node_execution_manager_test.go | 59 ++---- pkg/manager/impl/task_execution_manager.go | 37 ++-- .../impl/task_execution_manager_test.go | 42 ++-- pkg/rpc/adminservice/base.go | 14 +- pkg/runtime/application_config_provider.go | 7 + .../interfaces/application_configuration.go | 24 +++ .../mocks/mock_application_provider.go | 21 +- 14 files changed, 530 insertions(+), 262 deletions(-) create mode 100644 pkg/async/notifications/implementations/event_publisher.go create mode 100644 pkg/async/notifications/implementations/event_publisher_test.go diff --git a/flyteadmin_config.yaml b/flyteadmin_config.yaml index 11749e42d..e62521ada 100644 --- a/flyteadmin_config.yaml +++ b/flyteadmin_config.yaml @@ -78,6 +78,14 @@ notifications: Execution \"{{ name }}\" has {{ phase }} in \"{{ domain }}\". View details at http://example.com/projects/{{ project }}/domains/{{ domain }}/executions/{{ name }}. {{ error }} +externalEvents: + Enable: false + type: gcp + gcp: + projectId: "foo" + eventsPublisher: + topicName: "bar" + eventTypes: all Logger: show-source: true level: 6 diff --git a/pkg/async/notifications/factory.go b/pkg/async/notifications/factory.go index 83ee8c663..2293ff926 100644 --- a/pkg/async/notifications/factory.go +++ b/pkg/async/notifications/factory.go @@ -148,3 +148,53 @@ func NewNotificationsPublisher(config runtimeInterfaces.NotificationsConfig, sco return implementations.NewNoopPublish() } } + +func NewEventsPublisher(config runtimeInterfaces.ExternalEventsConfig, scope promutils.Scope) interfaces.Publisher { + if !config.Enable { + return implementations.NewNoopPublish() + } + reconnectAttempts := config.ReconnectAttempts + reconnectDelay := time.Duration(config.ReconnectDelaySeconds) * time.Second + switch config.Type { + case common.AWS: + snsConfig := gizmoAWS.SNSConfig{ + Topic: config.EventsPublisherConfig.TopicName, + } + snsConfig.Region = config.AWSConfig.Region + + var publisher pubsub.Publisher + var err error + err = async.Retry(reconnectAttempts, reconnectDelay, func() error { + publisher, err = gizmoAWS.NewPublisher(snsConfig) + return err + }) + + // Any persistent errors initiating Publisher with Amazon configurations results in a failed start up. + if err != nil { + panic(err) + } + return implementations.NewEventsPublisher(publisher, scope, config.EventsPublisherConfig.EventTypes) + case common.GCP: + pubsubConfig := gizmoGCP.Config{ + Topic: config.EventsPublisherConfig.TopicName, + } + pubsubConfig.ProjectID = config.GCPConfig.ProjectID + var publisher pubsub.MultiPublisher + var err error + err = async.Retry(reconnectAttempts, reconnectDelay, func() error { + publisher, err = gizmoGCP.NewPublisher(context.TODO(), pubsubConfig) + return err + }) + + if err != nil { + panic(err) + } + return implementations.NewEventsPublisher(publisher, scope, config.EventsPublisherConfig.EventTypes) + case common.Local: + fallthrough + default: + logger.Infof(context.Background(), + "Using default noop events publisher implementation for config type [%s]", config.Type) + return implementations.NewNoopPublish() + } +} diff --git a/pkg/async/notifications/implementations/event_publisher.go b/pkg/async/notifications/implementations/event_publisher.go new file mode 100644 index 000000000..9f306af8e --- /dev/null +++ b/pkg/async/notifications/implementations/event_publisher.go @@ -0,0 +1,105 @@ +package implementations + +import ( + "context" + + "k8s.io/apimachinery/pkg/util/sets" + + "github.com/lyft/flyteidl/gen/pb-go/flyteidl/admin" + + "github.com/lyft/flyteadmin/pkg/async/notifications/interfaces" + + "github.com/NYTimes/gizmo/pubsub" + "github.com/golang/protobuf/proto" + "github.com/lyft/flytestdlib/logger" + "github.com/lyft/flytestdlib/promutils" + "github.com/prometheus/client_golang/prometheus" +) + +type eventPublisherSystemMetrics struct { + Scope promutils.Scope + PublishTotal prometheus.Counter + PublishSuccess prometheus.Counter + PublishError prometheus.Counter +} + +// TODO: Add a counter that encompasses the publisher stats grouped by project and domain. +type EventPublisher struct { + pub pubsub.Publisher + systemMetrics eventPublisherSystemMetrics + events sets.String +} + +var taskExecutionReq admin.TaskExecutionEventRequest +var nodeExecutionReq admin.NodeExecutionEventRequest +var workflowExecutionReq admin.WorkflowExecutionEventRequest + +const ( + Task = "task" + Node = "node" + Workflow = "workflow" + AllTypes = "all" + AllTypesShort = "*" +) + +var supportedEvents = map[string]string{ + Task: proto.MessageName(&taskExecutionReq), + Node: proto.MessageName(&nodeExecutionReq), + Workflow: proto.MessageName(&workflowExecutionReq), +} + +// The key is the notification type as defined as an enum. +func (p *EventPublisher) Publish(ctx context.Context, notificationType string, msg proto.Message) error { + p.systemMetrics.PublishTotal.Inc() + + if !p.shouldPublishEvent(notificationType) { + return nil + } + logger.Debugf(ctx, "Publishing the following message [%+v]", msg) + + err := p.pub.Publish(ctx, notificationType, msg) + if err != nil { + p.systemMetrics.PublishError.Inc() + logger.Errorf(ctx, "Failed to publish a message with key [%s] and message [%s] and error: %v", notificationType, msg.String(), err) + } else { + p.systemMetrics.PublishSuccess.Inc() + } + return err +} + +func (p *EventPublisher) shouldPublishEvent(notificationType string) bool { + return p.events.Has(notificationType) +} + +func newEventPublisherSystemMetrics(scope promutils.Scope) eventPublisherSystemMetrics { + return eventPublisherSystemMetrics{ + Scope: scope, + PublishTotal: scope.MustNewCounter("event_publish_total", "overall count of publish messages"), + PublishSuccess: scope.MustNewCounter("event_publish_success", "success count of publish messages"), + PublishError: scope.MustNewCounter("event_publish_errors", "count of publish errors"), + } +} + +func NewEventsPublisher(pub pubsub.Publisher, scope promutils.Scope, eventTypes []string) interfaces.Publisher { + eventSet := sets.NewString() + + for _, event := range eventTypes { + if event == AllTypes || event == AllTypesShort { + for _, e := range supportedEvents { + eventSet = eventSet.Insert(e) + } + break + } + if e, found := supportedEvents[event]; found { + eventSet = eventSet.Insert(e) + } else { + logger.Errorf(context.Background(), "Unsupported event type [%s] in the config") + } + } + + return &EventPublisher{ + pub: pub, + systemMetrics: newEventPublisherSystemMetrics(scope.NewSubScope("events_publisher")), + events: eventSet, + } +} diff --git a/pkg/async/notifications/implementations/event_publisher_test.go b/pkg/async/notifications/implementations/event_publisher_test.go new file mode 100644 index 000000000..dbfed5607 --- /dev/null +++ b/pkg/async/notifications/implementations/event_publisher_test.go @@ -0,0 +1,173 @@ +package implementations + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/golang/protobuf/ptypes" + "github.com/lyft/flyteidl/gen/pb-go/flyteidl/admin" + "github.com/lyft/flyteidl/gen/pb-go/flyteidl/core" + "github.com/lyft/flyteidl/gen/pb-go/flyteidl/event" + + "github.com/NYTimes/gizmo/pubsub" + "github.com/NYTimes/gizmo/pubsub/pubsubtest" + "github.com/golang/protobuf/proto" + "github.com/lyft/flytestdlib/promutils" + "github.com/stretchr/testify/assert" +) + +var testEventPublisher pubsubtest.TestPublisher +var mockEventPublisher pubsub.Publisher = &testEventPublisher + +var executionID = core.WorkflowExecutionIdentifier{ + Project: "project", + Domain: "domain", + Name: "name", +} +var nodeExecutionID = core.NodeExecutionIdentifier{ + NodeId: "node id", + ExecutionId: &executionID, +} + +var taskID = &core.Identifier{ + ResourceType: core.ResourceType_TASK, + Project: "p", + Domain: "d", + Version: "v", + Name: "n", +} + +var occurredAt = time.Now().UTC() +var occurredAtProto, _ = ptypes.TimestampProto(occurredAt) + +var taskPhase = core.TaskExecution_RUNNING + +var retryAttempt = uint32(1) + +const requestID = "request id" + +var taskRequest = &admin.TaskExecutionEventRequest{ + RequestId: requestID, + Event: &event.TaskExecutionEvent{ + TaskId: taskID, + ParentNodeExecutionId: &nodeExecutionID, + RetryAttempt: retryAttempt, + Phase: taskPhase, + OccurredAt: occurredAtProto, + }, +} + +var nodeRequest = &admin.NodeExecutionEventRequest{ + RequestId: requestID, + Event: &event.NodeExecutionEvent{ + ProducerId: "propeller", + Id: &nodeExecutionID, + OccurredAt: occurredAtProto, + Phase: core.NodeExecution_RUNNING, + InputUri: "input uri", + }, +} + +var workflowRequest = &admin.WorkflowExecutionEventRequest{ + Event: &event.WorkflowExecutionEvent{ + Phase: core.WorkflowExecution_SUCCEEDED, + OutputResult: &event.WorkflowExecutionEvent_OutputUri{ + OutputUri: "somestring", + }, + ExecutionId: &executionID, + }, +} + +// This method should be invoked before every test around Publisher. +func initializeEventPublisher() { + testEventPublisher.Published = nil + testEventPublisher.GivenError = nil + testEventPublisher.FoundError = nil +} + +func TestNewEventsPublisher_EventTypes(t *testing.T) { + { + tests := []struct { + name string + eventTypes []string + events []proto.Message + shouldSendEvent []bool + expectedSendCnt int + }{ + {"eventTypes as workflow,node", []string{"workflow", "node"}, + []proto.Message{workflowRequest, nodeRequest, taskRequest}, + []bool{true, true, false}, + 2}, + {"eventTypes as workflow,task", []string{"workflow", "task"}, + []proto.Message{workflowRequest, nodeRequest, taskRequest}, + []bool{true, false, true}, + 2}, + {"eventTypes as workflow,task", []string{"node", "task"}, + []proto.Message{workflowRequest, nodeRequest, taskRequest}, + []bool{false, true, true}, + 2}, + {"eventTypes as task", []string{"task"}, + []proto.Message{taskRequest}, + []bool{true}, + 1}, + {"eventTypes as node", []string{"node"}, + []proto.Message{nodeRequest}, + []bool{true}, + 1}, + {"eventTypes as workflow", []string{"workflow"}, + []proto.Message{workflowRequest}, + []bool{true}, + 1}, + {"eventTypes as workflow", []string{"workflow"}, + []proto.Message{nodeRequest, taskRequest}, + []bool{false, false}, + 0}, + {"eventTypes as task", []string{"task"}, + []proto.Message{workflowRequest, nodeRequest}, + []bool{false, false}, + 0}, + {"eventTypes as node", []string{"node"}, + []proto.Message{workflowRequest, taskRequest}, + []bool{false, false}, + 0}, + {"eventTypes as all", []string{"all"}, + []proto.Message{workflowRequest, nodeRequest, taskRequest}, + []bool{true, true, true}, + 3}, + {"eventTypes as *", []string{"*"}, + []proto.Message{workflowRequest, nodeRequest, taskRequest}, + []bool{true, true, true}, + 3}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + initializeEventPublisher() + var currentEventPublisher = NewEventsPublisher(mockEventPublisher, promutils.NewTestScope(), test.eventTypes) + var cnt = 0 + for id, event := range test.events { + assert.Nil(t, currentEventPublisher.Publish(context.Background(), proto.MessageName(event), + event)) + if test.shouldSendEvent[id] { + assert.Equal(t, proto.MessageName(event), testEventPublisher.Published[cnt].Key) + marshalledData, err := proto.Marshal(event) + assert.Nil(t, err) + assert.Equal(t, marshalledData, testEventPublisher.Published[cnt].Body) + cnt++ + } + } + assert.Equal(t, test.expectedSendCnt, len(testEventPublisher.Published)) + }) + } + } +} + +func TestEventPublisher_PublishError(t *testing.T) { + initializeEventPublisher() + currentEventPublisher := NewEventsPublisher(mockEventPublisher, promutils.NewTestScope(), []string{"*"}) + var publishError = errors.New("publish() returns an error") + testEventPublisher.GivenError = publishError + assert.Equal(t, publishError, currentEventPublisher.Publish(context.Background(), + proto.MessageName(taskRequest), taskRequest)) +} diff --git a/pkg/manager/impl/execution_manager.go b/pkg/manager/impl/execution_manager.go index dfe85367e..1453fb88c 100644 --- a/pkg/manager/impl/execution_manager.go +++ b/pkg/manager/impl/execution_manager.go @@ -64,6 +64,7 @@ type executionSystemMetrics struct { SpecSizeBytes prometheus.Summary ClosureSizeBytes prometheus.Summary AcceptanceDelay prometheus.Summary + PublishEventError prometheus.Counter } type executionUserMetrics struct { @@ -89,6 +90,7 @@ type ExecutionManager struct { namedEntityManager interfaces.NamedEntityInterface resourceManager interfaces.ResourceInterface qualityOfServiceAllocator executions.QualityOfServiceAllocator + eventPublisher notificationInterfaces.Publisher } func getExecutionContext(ctx context.Context, id *core.WorkflowExecutionIdentifier) context.Context { @@ -1004,6 +1006,10 @@ func (m *ExecutionManager) CreateWorkflowEvent(ctx context.Context, request admi return nil, err } } + if err := m.eventPublisher.Publish(ctx, proto.MessageName(&request), &request); err != nil { + m.systemMetrics.PublishEventError.Inc() + logger.Infof(ctx, "error publishing event [%+v] with err: [%v]", request.RequestId, err) + } m.systemMetrics.ExecutionEventsCreated.Inc() return &admin.WorkflowExecutionEventResponse{}, nil @@ -1326,20 +1332,12 @@ func newExecutionSystemMetrics(scope promutils.Scope) executionSystemMetrics { ClosureSizeBytes: scope.MustNewSummary("closure_size_bytes", "size in bytes of serialized execution closure"), AcceptanceDelay: scope.MustNewSummary("acceptance_delay", "delay in seconds from when an execution was requested to be created and when it actually was"), + PublishEventError: scope.MustNewCounter("publish_event_error", + "overall count of publish event errors when invoking publish()"), } } -func NewExecutionManager( - db repositories.RepositoryInterface, - config runtimeInterfaces.Configuration, - storageClient *storage.DataStore, - workflowExecutor workflowengineInterfaces.Executor, - systemScope promutils.Scope, - userScope promutils.Scope, - publisher notificationInterfaces.Publisher, - urlData dataInterfaces.RemoteURLInterface, - workflowManager interfaces.WorkflowInterface, - namedEntityManager interfaces.NamedEntityInterface) interfaces.ExecutionInterface { +func NewExecutionManager(db repositories.RepositoryInterface, config runtimeInterfaces.Configuration, storageClient *storage.DataStore, workflowExecutor workflowengineInterfaces.Executor, systemScope promutils.Scope, userScope promutils.Scope, publisher notificationInterfaces.Publisher, urlData dataInterfaces.RemoteURLInterface, workflowManager interfaces.WorkflowInterface, namedEntityManager interfaces.NamedEntityInterface, eventPublisher notificationInterfaces.Publisher) interfaces.ExecutionInterface { queueAllocator := executions.NewQueueAllocator(config, db) systemMetrics := newExecutionSystemMetrics(systemScope) @@ -1369,6 +1367,7 @@ func NewExecutionManager( namedEntityManager: namedEntityManager, resourceManager: resourceManager, qualityOfServiceAllocator: executions.NewQualityOfServiceAllocator(config, resourceManager), + eventPublisher: eventPublisher, } } diff --git a/pkg/manager/impl/execution_manager_test.go b/pkg/manager/impl/execution_manager_test.go index f0e5d85ba..1e5dba15e 100644 --- a/pkg/manager/impl/execution_manager_test.go +++ b/pkg/manager/impl/execution_manager_test.go @@ -256,9 +256,7 @@ func TestCreateExecution(t *testing.T) { mockConfig := getMockExecutionsConfigProvider() mockConfig.(*runtimeMocks.MockConfigurationProvider).AddQualityOfServiceConfiguration(qosProvider) - execManager := NewExecutionManager( - repository, mockConfig, getMockStorageForExecTest(context.Background()), mockExecutor, - mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil) + execManager := NewExecutionManager(repository, mockConfig, getMockStorageForExecTest(context.Background()), mockExecutor, mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, &mockPublisher) request := testutils.GetExecutionRequest() request.Spec.Metadata = &admin.ExecutionMetadata{ Principal: "unused - populated from authenticated context", @@ -332,9 +330,7 @@ func TestCreateExecutionFromWorkflowNode(t *testing.T) { }, ) - execManager := NewExecutionManager( - repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), - mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil) + execManager := NewExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil) request := testutils.GetExecutionRequest() request.Spec.Metadata = &admin.ExecutionMetadata{ Mode: admin.ExecutionMetadata_CHILD_WORKFLOW, @@ -372,8 +368,7 @@ func TestCreateExecution_NoAssignedName(t *testing.T) { Cluster: testCluster, }, nil }) - execManager := NewExecutionManager( - repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockExecutor, mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil) + execManager := NewExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockExecutor, mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil) request := testutils.GetExecutionRequest() request.Name = "" response, err := execManager.CreateExecution(context.Background(), request, requestedAt) @@ -421,8 +416,7 @@ func TestCreateExecution_TaggedQueue(t *testing.T) { Cluster: testCluster, }, nil }) - execManager := NewExecutionManager( - repository, configProvider, getMockStorageForExecTest(context.Background()), mockExecutor, mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil) + execManager := NewExecutionManager(repository, configProvider, getMockStorageForExecTest(context.Background()), mockExecutor, mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil) request := testutils.GetExecutionRequest() response, err := execManager.CreateExecution(context.Background(), request, requestedAt) @@ -438,9 +432,7 @@ func TestCreateExecution_TaggedQueue(t *testing.T) { func TestCreateExecutionValidationError(t *testing.T) { repository := getMockRepositoryForExecTest() setDefaultLpCallbackForExecTest(repository) - execManager := NewExecutionManager( - repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), - mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil) + execManager := NewExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil) request := testutils.GetExecutionRequest() request.Domain = "" @@ -452,9 +444,7 @@ func TestCreateExecutionValidationError(t *testing.T) { func TestCreateExecution_InvalidLpIdentifier(t *testing.T) { repository := getMockRepositoryForExecTest() setDefaultLpCallbackForExecTest(repository) - execManager := NewExecutionManager( - repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), - mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil) + execManager := NewExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil) request := testutils.GetExecutionRequest() request.Spec.LaunchPlan = nil @@ -466,9 +456,7 @@ func TestCreateExecution_InvalidLpIdentifier(t *testing.T) { func TestCreateExecutionInCompatibleInputs(t *testing.T) { repository := getMockRepositoryForExecTest() setDefaultLpCallbackForExecTest(repository) - execManager := NewExecutionManager( - repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), - mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil) + execManager := NewExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil) request := testutils.GetExecutionRequest() request.Inputs = &core.LiteralMap{ @@ -492,8 +480,7 @@ func TestCreateExecutionPropellerFailure(t *testing.T) { return nil, expectedErr } mockExecutor.(*workflowengineMocks.MockExecutor).SetExecuteWorkflowCallback(createFunc) - execManager := NewExecutionManager( - repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockExecutor, mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil) + execManager := NewExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockExecutor, mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil) request := testutils.GetExecutionRequest() @@ -512,9 +499,7 @@ func TestCreateExecutionDatabaseFailure(t *testing.T) { } repository.ExecutionRepo().(*repositoryMocks.MockExecutionRepo).SetCreateCallback(exCreateFunc) - execManager := NewExecutionManager( - repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), - mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil) + execManager := NewExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil) request := testutils.GetExecutionRequest() response, err := execManager.CreateExecution(context.Background(), request, requestedAt) @@ -577,9 +562,7 @@ func TestCreateExecutionVerifyDbModel(t *testing.T) { } repository.ExecutionRepo().(*repositoryMocks.MockExecutionRepo).SetCreateCallback(exCreateFunc) - execManager := NewExecutionManager( - repository, getMockExecutionsConfigProvider(), storageClient, workflowengineMocks.NewMockExecutor(), - mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil) + execManager := NewExecutionManager(repository, getMockExecutionsConfigProvider(), storageClient, workflowengineMocks.NewMockExecutor(), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil) execManager.(*ExecutionManager)._clock = mockClock @@ -617,9 +600,7 @@ func TestCreateExecutionDefaultNotifications(t *testing.T) { return nil } repository.ExecutionRepo().(*repositoryMocks.MockExecutionRepo).SetCreateCallback(exCreateFunc) - execManager := NewExecutionManager( - repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), mockScope.NewTestScope(), - mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil) + execManager := NewExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil) response, err := execManager.CreateExecution(context.Background(), request, requestedAt) assert.Nil(t, err) @@ -653,9 +634,7 @@ func TestCreateExecutionDisableNotifications(t *testing.T) { return nil } repository.ExecutionRepo().(*repositoryMocks.MockExecutionRepo).SetCreateCallback(exCreateFunc) - execManager := NewExecutionManager( - repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), mockScope.NewTestScope(), - mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil) + execManager := NewExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil) response, err := execManager.CreateExecution(context.Background(), request, requestedAt) assert.Nil(t, err) @@ -720,9 +699,7 @@ func TestCreateExecutionNoNotifications(t *testing.T) { return nil } repository.ExecutionRepo().(*repositoryMocks.MockExecutionRepo).SetCreateCallback(exCreateFunc) - execManager := NewExecutionManager( - repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), - mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil) + execManager := NewExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil) response, err := execManager.CreateExecution(context.Background(), request, requestedAt) assert.Nil(t, err) @@ -751,9 +728,7 @@ func TestCreateExecutionDynamicLabelsAndAnnotations(t *testing.T) { Cluster: testCluster, }, nil }) - execManager := NewExecutionManager( - repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockExecutor, - mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil) + execManager := NewExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockExecutor, mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil) request := testutils.GetExecutionRequest() request.Spec.Labels = &admin.Labels{ Values: map[string]string{ @@ -833,9 +808,7 @@ func TestRelaunchExecution(t *testing.T) { // Set up mocks. repository := getMockRepositoryForExecTest() setDefaultLpCallbackForExecTest(repository) - execManager := NewExecutionManager( - repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), - mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil) + execManager := NewExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil) startTime := time.Now() startTimeProto, _ := ptypes.TimestampProto(startTime) existingClosure := admin.ExecutionClosure{ @@ -892,9 +865,7 @@ func TestRelaunchExecution_GetExistingFailure(t *testing.T) { // Set up mocks. repository := getMockRepositoryForExecTest() setDefaultLpCallbackForExecTest(repository) - execManager := NewExecutionManager( - repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), - mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil) + execManager := NewExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil) expectedErr := errors.New("expected error") repository.ExecutionRepo().(*repositoryMocks.MockExecutionRepo).SetGetCallback( @@ -928,9 +899,7 @@ func TestRelaunchExecution_CreateFailure(t *testing.T) { // Set up mocks. repository := getMockRepositoryForExecTest() setDefaultLpCallbackForExecTest(repository) - execManager := NewExecutionManager( - repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), - mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil) + execManager := NewExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil) startTime := time.Now() startTimeProto, _ := ptypes.TimestampProto(startTime) existingClosure := admin.ExecutionClosure{ @@ -1015,9 +984,7 @@ func TestCreateWorkflowEvent(t *testing.T) { return nil } repository.ExecutionRepo().(*repositoryMocks.MockExecutionRepo).SetUpdateCallback(updateExecutionFunc) - execManager := NewExecutionManager( - repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), - mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil) + execManager := NewExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, &mockPublisher) resp, err := execManager.CreateWorkflowEvent(context.Background(), admin.WorkflowExecutionEventRequest{ RequestId: "1", Event: &event.WorkflowExecutionEvent{ @@ -1055,9 +1022,7 @@ func TestCreateWorkflowEvent_TerminalState(t *testing.T) { return nil } repository.ExecutionRepo().(*repositoryMocks.MockExecutionRepo).SetUpdateCallback(updateExecutionFunc) - execManager := NewExecutionManager( - repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), - mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil) + execManager := NewExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil) resp, err := execManager.CreateWorkflowEvent(context.Background(), admin.WorkflowExecutionEventRequest{ RequestId: "1", @@ -1112,9 +1077,7 @@ func TestCreateWorkflowEvent_StartedRunning(t *testing.T) { return nil } repository.ExecutionRepo().(*repositoryMocks.MockExecutionRepo).SetUpdateCallback(updateExecutionFunc) - execManager := NewExecutionManager( - repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), - mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil) + execManager := NewExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, &mockPublisher) occurredAtTimestamp, _ := ptypes.TimestampProto(occurredAt) resp, err := execManager.CreateWorkflowEvent(context.Background(), admin.WorkflowExecutionEventRequest{ RequestId: "1", @@ -1153,9 +1116,7 @@ func TestCreateWorkflowEvent_DuplicateRunning(t *testing.T) { }, ) - execManager := NewExecutionManager( - repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), - mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil) + execManager := NewExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil) occurredAtTimestamp, _ := ptypes.TimestampProto(occurredAt) resp, err := execManager.CreateWorkflowEvent(context.Background(), admin.WorkflowExecutionEventRequest{ RequestId: "1", @@ -1196,9 +1157,7 @@ func TestCreateWorkflowEvent_InvalidPhaseChange(t *testing.T) { }, ) - execManager := NewExecutionManager( - repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), - mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil) + execManager := NewExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil) occurredAtTimestamp, _ := ptypes.TimestampProto(occurredAt) resp, err := execManager.CreateWorkflowEvent(context.Background(), admin.WorkflowExecutionEventRequest{ RequestId: "1", @@ -1232,9 +1191,7 @@ func TestCreateWorkflowEvent_InvalidEvent(t *testing.T) { return nil } repository.ExecutionRepo().(*repositoryMocks.MockExecutionRepo).SetUpdateCallback(updateExecutionFunc) - execManager := NewExecutionManager( - repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), - mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil) + execManager := NewExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil) resp, err := execManager.CreateWorkflowEvent(context.Background(), admin.WorkflowExecutionEventRequest{ RequestId: "1", Event: &event.WorkflowExecutionEvent{ @@ -1262,9 +1219,7 @@ func TestCreateWorkflowEvent_UpdateModelError(t *testing.T) { Message: "bar baz", } - execManager := NewExecutionManager( - repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), - mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil) + execManager := NewExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil) resp, err := execManager.CreateWorkflowEvent(context.Background(), admin.WorkflowExecutionEventRequest{ RequestId: "1", Event: &event.WorkflowExecutionEvent{ @@ -1296,9 +1251,7 @@ func TestCreateWorkflowEvent_DatabaseGetError(t *testing.T) { Code: "foo", Message: "bar baz", } - execManager := NewExecutionManager( - repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), - mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil) + execManager := NewExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil) resp, err := execManager.CreateWorkflowEvent(context.Background(), admin.WorkflowExecutionEventRequest{ RequestId: "1", Event: &event.WorkflowExecutionEvent{ @@ -1332,9 +1285,7 @@ func TestCreateWorkflowEvent_DatabaseUpdateError(t *testing.T) { return expectedErr } repository.ExecutionRepo().(*repositoryMocks.MockExecutionRepo).SetUpdateCallback(updateExecutionFunc) - execManager := NewExecutionManager( - repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), - mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil) + execManager := NewExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil) resp, err := execManager.CreateWorkflowEvent(context.Background(), admin.WorkflowExecutionEventRequest{ RequestId: "1", Event: &event.WorkflowExecutionEvent{ @@ -1373,9 +1324,7 @@ func TestGetExecution(t *testing.T) { }, nil } repository.ExecutionRepo().(*repositoryMocks.MockExecutionRepo).SetGetCallback(executionGetFunc) - execManager := NewExecutionManager( - repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), - mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil) + execManager := NewExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil) execution, err := execManager.GetExecution(context.Background(), admin.WorkflowExecutionGetRequest{ Id: &executionIdentifier, }) @@ -1396,9 +1345,7 @@ func TestGetExecution_DatabaseError(t *testing.T) { return models.Execution{}, expectedErr } repository.ExecutionRepo().(*repositoryMocks.MockExecutionRepo).SetGetCallback(executionGetFunc) - execManager := NewExecutionManager( - repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), - mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil) + execManager := NewExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil) execution, err := execManager.GetExecution(context.Background(), admin.WorkflowExecutionGetRequest{ Id: &executionIdentifier, }) @@ -1428,9 +1375,7 @@ func TestGetExecution_TransformerError(t *testing.T) { }, nil } repository.ExecutionRepo().(*repositoryMocks.MockExecutionRepo).SetGetCallback(executionGetFunc) - execManager := NewExecutionManager( - repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), - mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil) + execManager := NewExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil) execution, err := execManager.GetExecution(context.Background(), admin.WorkflowExecutionGetRequest{ Id: &executionIdentifier, }) @@ -1490,9 +1435,7 @@ func TestListExecutions(t *testing.T) { }, nil } repository.ExecutionRepo().(*repositoryMocks.MockExecutionRepo).SetListCallback(executionListFunc) - execManager := NewExecutionManager( - repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), - mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil) + execManager := NewExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil) executionList, err := execManager.ListExecutions(context.Background(), admin.ResourceListRequest{ Id: &admin.NamedEntityIdentifier{ @@ -1523,10 +1466,7 @@ func TestListExecutions(t *testing.T) { } func TestListExecutions_MissingParameters(t *testing.T) { - execManager := NewExecutionManager( - repositoryMocks.NewMockRepository(), - getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), - mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil) + execManager := NewExecutionManager(repositoryMocks.NewMockRepository(), getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil) _, err := execManager.ListExecutions(context.Background(), admin.ResourceListRequest{ Id: &admin.NamedEntityIdentifier{ Domain: domainValue, @@ -1563,9 +1503,7 @@ func TestListExecutions_DatabaseError(t *testing.T) { return interfaces.ExecutionCollectionOutput{}, expectedErr } repository.ExecutionRepo().(*repositoryMocks.MockExecutionRepo).SetListCallback(executionListFunc) - execManager := NewExecutionManager( - repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), - mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil) + execManager := NewExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil) _, err := execManager.ListExecutions(context.Background(), admin.ResourceListRequest{ Id: &admin.NamedEntityIdentifier{ Project: projectValue, @@ -1596,9 +1534,7 @@ func TestListExecutions_TransformerError(t *testing.T) { }, nil } repository.ExecutionRepo().(*repositoryMocks.MockExecutionRepo).SetListCallback(executionListFunc) - execManager := NewExecutionManager( - repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), - mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil) + execManager := NewExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil) executionList, err := execManager.ListExecutions(context.Background(), admin.ResourceListRequest{ Id: &admin.NamedEntityIdentifier{ @@ -1905,9 +1841,7 @@ func TestTerminateExecution(t *testing.T) { }, input.ExecutionID)) return nil }) - execManager := NewExecutionManager( - repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockExecutor, - mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil) + execManager := NewExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockExecutor, mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil) ctx := context.WithValue(context.Background(), auth.PrincipalContextKey, principal) resp, err := execManager.TerminateExecution(ctx, admin.ExecutionTerminateRequest{ @@ -1937,9 +1871,7 @@ func TestTerminateExecution_PropellerError(t *testing.T) { t.Fatal("update should not be called when propeller fails to terminate an execution") return nil }) - execManager := NewExecutionManager( - repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockExecutor, - mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil) + execManager := NewExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockExecutor, mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil) resp, err := execManager.TerminateExecution(context.Background(), admin.ExecutionTerminateRequest{ Id: &core.WorkflowExecutionIdentifier{ @@ -1966,9 +1898,7 @@ func TestTerminateExecution_DatabaseError(t *testing.T) { } repository.ExecutionRepo().(*repositoryMocks.MockExecutionRepo).SetUpdateExecutionCallback(updateExecutionFunc) - execManager := NewExecutionManager( - repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), - mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil) + execManager := NewExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil) resp, err := execManager.TerminateExecution(context.Background(), admin.ExecutionTerminateRequest{ Id: &core.WorkflowExecutionIdentifier{ Project: "project", @@ -2056,9 +1986,7 @@ func TestGetExecutionData(t *testing.T) { } repository.ExecutionRepo().(*repositoryMocks.MockExecutionRepo).SetGetCallback(executionGetFunc) - execManager := NewExecutionManager( - repository, getMockExecutionsConfigProvider(), mockStorage, workflowengineMocks.NewMockExecutor(), - mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil) + execManager := NewExecutionManager(repository, getMockExecutionsConfigProvider(), mockStorage, workflowengineMocks.NewMockExecutor(), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil) dataResponse, err := execManager.GetExecutionData(context.Background(), admin.WorkflowExecutionGetDataRequest{ Id: &executionIdentifier, }) @@ -2087,9 +2015,7 @@ func TestAddLabelsAndAnnotationsRuntimeLimitsObserved(t *testing.T) { configProvider := getMockExecutionsConfigProvider() configProvider.(*runtimeMocks.MockConfigurationProvider).AddRegistrationValidationConfiguration( mockRegistrationValidationConfig) - execManager := NewExecutionManager( - repository, configProvider, getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), - mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil) + execManager := NewExecutionManager(repository, configProvider, getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil) request := testutils.GetExecutionRequest() request.Spec.Labels = &admin.Labels{ Values: map[string]string{ @@ -2153,9 +2079,7 @@ func TestAddPluginOverrides(t *testing.T) { } partiallyPopulatedInputs := workflowengineInterfaces.ExecuteWorkflowInput{} - execManager := NewExecutionManager( - db, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), - mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil) + execManager := NewExecutionManager(db, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil) taskPluginOverrides, err := execManager.(*ExecutionManager).addPluginOverrides( context.Background(), executionID, workflowName, launchPlanName) @@ -2186,9 +2110,7 @@ func TestPluginOverrides_ResourceGetFailure(t *testing.T) { models.Resource, error) { return models.Resource{}, flyteAdminErrors.NewFlyteAdminErrorf(codes.Aborted, "uh oh") } - execManager := NewExecutionManager( - db, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), - mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil) + execManager := NewExecutionManager(db, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil) _, err := execManager.(*ExecutionManager).addPluginOverrides( context.Background(), executionID, workflowName, launchPlanName) @@ -2217,9 +2139,7 @@ func TestGetExecution_Legacy(t *testing.T) { }, nil } repository.ExecutionRepo().(*repositoryMocks.MockExecutionRepo).SetGetCallback(executionGetFunc) - execManager := NewExecutionManager( - repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), - mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil) + execManager := NewExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil) execution, err := execManager.GetExecution(context.Background(), admin.WorkflowExecutionGetRequest{ Id: &executionIdentifier, }) @@ -2254,9 +2174,7 @@ func TestGetExecution_LegacyClient_OffloadedData(t *testing.T) { } repository.ExecutionRepo().(*repositoryMocks.MockExecutionRepo).SetGetCallback(executionGetFunc) storageClient := getMockStorageForExecTest(context.Background()) - execManager := NewExecutionManager( - repository, getMockExecutionsConfigProvider(), storageClient, workflowengineMocks.NewMockExecutor(), - mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil) + execManager := NewExecutionManager(repository, getMockExecutionsConfigProvider(), storageClient, workflowengineMocks.NewMockExecutor(), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil) _ = storageClient.WriteProtobuf(context.Background(), storage.DataReference(shared.UserInputs), storage.Options{}, getLegacySpec().Inputs) _ = storageClient.WriteProtobuf(context.Background(), storage.DataReference(shared.Inputs), storage.Options{}, getLegacyClosure().ComputedInputs) execution, err := execManager.GetExecution(context.Background(), admin.WorkflowExecutionGetRequest{ @@ -2316,9 +2234,7 @@ func TestGetExecutionData_LegacyModel(t *testing.T) { repository.ExecutionRepo().(*repositoryMocks.MockExecutionRepo).SetGetCallback(executionGetFunc) storageClient := getMockStorageForExecTest(context.Background()) - execManager := NewExecutionManager( - repository, getMockExecutionsConfigProvider(), storageClient, workflowengineMocks.NewMockExecutor(), - mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil) + execManager := NewExecutionManager(repository, getMockExecutionsConfigProvider(), storageClient, workflowengineMocks.NewMockExecutor(), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil) dataResponse, err := execManager.GetExecutionData(context.Background(), admin.WorkflowExecutionGetDataRequest{ Id: &executionIdentifier, }) @@ -2363,9 +2279,7 @@ func TestCreateExecution_LegacyClient(t *testing.T) { Cluster: testCluster, }, nil }) - execManager := NewExecutionManager( - repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockExecutor, - mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil) + execManager := NewExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockExecutor, mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil) response, err := execManager.CreateExecution(context.Background(), *getLegacyExecutionRequest(), requestedAt) assert.Nil(t, err) @@ -2381,9 +2295,7 @@ func TestRelaunchExecution_LegacyModel(t *testing.T) { repository := getMockRepositoryForExecTest() setDefaultLpCallbackForExecTest(repository) storageClient := getMockStorageForExecTest(context.Background()) - execManager := NewExecutionManager( - repository, getMockExecutionsConfigProvider(), storageClient, workflowengineMocks.NewMockExecutor(), - mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil) + execManager := NewExecutionManager(repository, getMockExecutionsConfigProvider(), storageClient, workflowengineMocks.NewMockExecutor(), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil) startTime := time.Now() startTimeProto, _ := ptypes.TimestampProto(startTime) existingClosure := getLegacyClosure() @@ -2494,9 +2406,7 @@ func TestListExecutions_LegacyModel(t *testing.T) { }, nil } repository.ExecutionRepo().(*repositoryMocks.MockExecutionRepo).SetListCallback(executionListFunc) - execManager := NewExecutionManager( - repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), - mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil) + execManager := NewExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil) executionList, err := execManager.ListExecutions(context.Background(), admin.ResourceListRequest{ Id: &admin.NamedEntityIdentifier{ @@ -2698,9 +2608,7 @@ func TestSetDefaults(t *testing.T) { mockConfig := runtimeMocks.NewMockConfigurationProvider( testutils.GetApplicationConfigWithDefaultDomains(), nil, nil, &taskConfig, runtimeMocks.NewMockWhitelistConfiguration(), nil) - execManager := NewExecutionManager( - repositoryMocks.NewMockRepository(), mockConfig, getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), - mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil) + execManager := NewExecutionManager(repositoryMocks.NewMockRepository(), mockConfig, getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil) execManager.(*ExecutionManager).setCompiledTaskDefaults(context.Background(), task, "workflow") assert.True(t, proto.Equal( &core.Container{ @@ -2767,9 +2675,7 @@ func TestSetDefaults_MissingDefaults(t *testing.T) { mockConfig := runtimeMocks.NewMockConfigurationProvider( testutils.GetApplicationConfigWithDefaultDomains(), nil, nil, &taskConfig, runtimeMocks.NewMockWhitelistConfiguration(), nil) - execManager := NewExecutionManager( - repositoryMocks.NewMockRepository(), mockConfig, getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), - mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil) + execManager := NewExecutionManager(repositoryMocks.NewMockRepository(), mockConfig, getMockStorageForExecTest(context.Background()), workflowengineMocks.NewMockExecutor(), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil) execManager.(*ExecutionManager).setCompiledTaskDefaults(context.Background(), task, "workflow") assert.True(t, proto.Equal( &core.Container{ @@ -2952,9 +2858,7 @@ func TestCreateSingleTaskExecution(t *testing.T) { getMockWorkflowConfigProvider(), getMockWorkflowCompiler(), mockStorage, storagePrefix, mockScope.NewTestScope()) namedEntityManager := NewNamedEntityManager(repository, getMockConfigForNETest(), mockScope.NewTestScope()) - execManager := NewExecutionManager( - repository, getMockExecutionsConfigProvider(), mockStorage, workflowengineMocks.NewMockExecutor(), - mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, workflowManager, namedEntityManager) + execManager := NewExecutionManager(repository, getMockExecutionsConfigProvider(), mockStorage, workflowengineMocks.NewMockExecutor(), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, workflowManager, namedEntityManager, nil) request := admin.ExecutionCreateRequest{ Project: "flytekit", Domain: "production", diff --git a/pkg/manager/impl/node_execution_manager.go b/pkg/manager/impl/node_execution_manager.go index 3e37e9c87..87f2dbbd6 100644 --- a/pkg/manager/impl/node_execution_manager.go +++ b/pkg/manager/impl/node_execution_manager.go @@ -4,6 +4,9 @@ import ( "context" "strconv" + "github.com/golang/protobuf/proto" + notificationInterfaces "github.com/lyft/flyteadmin/pkg/async/notifications/interfaces" + "github.com/lyft/flytestdlib/storage" "github.com/lyft/flytestdlib/contextutils" @@ -43,14 +46,16 @@ type nodeExecutionMetrics struct { ClosureSizeBytes prometheus.Summary NodeExecutionInputBytes prometheus.Summary NodeExecutionOutputBytes prometheus.Summary + PublishEventError prometheus.Counter } type NodeExecutionManager struct { - db repositories.RepositoryInterface - config runtimeInterfaces.Configuration - storageClient *storage.DataStore - metrics nodeExecutionMetrics - urlData dataInterfaces.RemoteURLInterface + db repositories.RepositoryInterface + config runtimeInterfaces.Configuration + storageClient *storage.DataStore + metrics nodeExecutionMetrics + urlData dataInterfaces.RemoteURLInterface + eventPublisher notificationInterfaces.Publisher } type updateNodeExecutionStatus int @@ -232,6 +237,11 @@ func (m *NodeExecutionManager) CreateNodeEvent(ctx context.Context, request admi } m.metrics.NodeExecutionEventsCreated.Inc() + if err := m.eventPublisher.Publish(ctx, proto.MessageName(&request), &request); err != nil { + m.metrics.PublishEventError.Inc() + logger.Infof(ctx, "error publishing event [%+v] with err: [%v]", request.RequestId, err) + } + return &admin.NodeExecutionEventResponse{}, nil } @@ -427,9 +437,7 @@ func (m *NodeExecutionManager) GetNodeExecutionData( return response, nil } -func NewNodeExecutionManager( - db repositories.RepositoryInterface, config runtimeInterfaces.Configuration, storageClient *storage.DataStore, - scope promutils.Scope, urlData dataInterfaces.RemoteURLInterface) interfaces.NodeExecutionInterface { +func NewNodeExecutionManager(db repositories.RepositoryInterface, config runtimeInterfaces.Configuration, storageClient *storage.DataStore, scope promutils.Scope, urlData dataInterfaces.RemoteURLInterface, eventPublisher notificationInterfaces.Publisher) interfaces.NodeExecutionInterface { metrics := nodeExecutionMetrics{ Scope: scope, ActiveNodeExecutions: scope.MustNewGauge("active_node_executions", @@ -448,12 +456,15 @@ func NewNodeExecutionManager( "size in bytes of serialized node execution inputs"), NodeExecutionOutputBytes: scope.MustNewSummary("output_size_bytes", "size in bytes of serialized node execution outputs"), + PublishEventError: scope.MustNewCounter("publish_event_error", + "overall count of publish event errors when invoking publish()"), } return &NodeExecutionManager{ - db: db, - config: config, - storageClient: storageClient, - metrics: metrics, - urlData: urlData, + db: db, + config: config, + storageClient: storageClient, + metrics: metrics, + urlData: urlData, + eventPublisher: eventPublisher, } } diff --git a/pkg/manager/impl/node_execution_manager_test.go b/pkg/manager/impl/node_execution_manager_test.go index 676baa495..aa0306ae0 100644 --- a/pkg/manager/impl/node_execution_manager_test.go +++ b/pkg/manager/impl/node_execution_manager_test.go @@ -135,7 +135,8 @@ func TestCreateNodeEvent(t *testing.T) { return nil }) nodeExecManager := NewNodeExecutionManager(repository, getMockExecutionsConfigProvider(), - getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockNodeExecutionRemoteURL) + getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockNodeExecutionRemoteURL, + &mockPublisher) resp, err := nodeExecManager.CreateNodeEvent(context.Background(), request) assert.Nil(t, err) assert.NotNil(t, resp) @@ -191,7 +192,7 @@ func TestCreateNodeEvent_Update(t *testing.T) { return nil }) nodeExecManager := NewNodeExecutionManager(repository, getMockExecutionsConfigProvider(), - getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockNodeExecutionRemoteURL) + getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockNodeExecutionRemoteURL, &mockPublisher) resp, err := nodeExecManager.CreateNodeEvent(context.Background(), request) assert.Nil(t, err) assert.NotNil(t, resp) @@ -204,8 +205,7 @@ func TestCreateNodeEvent_MissingExecution(t *testing.T) { func(ctx context.Context, input interfaces.GetResourceInput) (models.Execution, error) { return models.Execution{}, expectedErr }) - nodeExecManager := NewNodeExecutionManager(repository, getMockExecutionsConfigProvider(), - getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockNodeExecutionRemoteURL) + nodeExecManager := NewNodeExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockNodeExecutionRemoteURL, &mockPublisher) resp, err := nodeExecManager.CreateNodeEvent(context.Background(), request) assert.EqualError(t, err, "failed to get existing execution id: [project:\"project\""+ " domain:\"domain\" name:\"name\" ] with err: expected error") @@ -225,8 +225,7 @@ func TestCreateNodeEvent_CreateDatabaseError(t *testing.T) { func(ctx context.Context, event *models.NodeExecutionEvent, input *models.NodeExecution) error { return expectedErr }) - nodeExecManager := NewNodeExecutionManager(repository, getMockExecutionsConfigProvider(), - getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockNodeExecutionRemoteURL) + nodeExecManager := NewNodeExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockNodeExecutionRemoteURL, nil) resp, err := nodeExecManager.CreateNodeEvent(context.Background(), request) assert.EqualError(t, err, expectedErr.Error()) assert.Nil(t, resp) @@ -261,8 +260,7 @@ func TestCreateNodeEvent_UpdateDatabaseError(t *testing.T) { func(ctx context.Context, event *models.NodeExecutionEvent, nodeExecution *models.NodeExecution) error { return expectedErr }) - nodeExecManager := NewNodeExecutionManager(repository, getMockExecutionsConfigProvider(), - getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockNodeExecutionRemoteURL) + nodeExecManager := NewNodeExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockNodeExecutionRemoteURL, nil) resp, err := nodeExecManager.CreateNodeEvent(context.Background(), request) assert.EqualError(t, err, expectedErr.Error()) assert.Nil(t, resp) @@ -291,8 +289,7 @@ func TestCreateNodeEvent_UpdateTerminalEventError(t *testing.T) { StartedAt: &occurredAt, }, nil }) - nodeExecManager := NewNodeExecutionManager(repository, getMockExecutionsConfigProvider(), - getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockNodeExecutionRemoteURL) + nodeExecManager := NewNodeExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockNodeExecutionRemoteURL, nil) resp, err := nodeExecManager.CreateNodeEvent(context.Background(), request) assert.Nil(t, resp) assert.NotNil(t, err) @@ -327,8 +324,7 @@ func TestCreateNodeEvent_UpdateDuplicateEventError(t *testing.T) { StartedAt: &occurredAt, }, nil }) - nodeExecManager := NewNodeExecutionManager(repository, getMockExecutionsConfigProvider(), - getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockNodeExecutionRemoteURL) + nodeExecManager := NewNodeExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockNodeExecutionRemoteURL, nil) resp, err := nodeExecManager.CreateNodeEvent(context.Background(), request) assert.Equal(t, codes.AlreadyExists, err.(flyteAdminErrors.FlyteAdminError).Code()) assert.Nil(t, resp) @@ -341,8 +337,7 @@ func TestCreateNodeEvent_FirstEventIsTerminal(t *testing.T) { func(ctx context.Context, input interfaces.GetNodeExecutionInput) (models.NodeExecution, error) { return models.NodeExecution{}, flyteAdminErrors.NewFlyteAdminError(codes.NotFound, "foo") }) - nodeExecManager := NewNodeExecutionManager(repository, getMockExecutionsConfigProvider(), - getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockNodeExecutionRemoteURL) + nodeExecManager := NewNodeExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockNodeExecutionRemoteURL, &mockPublisher) succeededRequest := admin.NodeExecutionEventRequest{ RequestId: "request id", Event: &event.NodeExecutionEvent{ @@ -403,8 +398,7 @@ func TestGetNodeExecution(t *testing.T) { NodeExecutionMetadata: metadataBytes, }, nil }) - nodeExecManager := NewNodeExecutionManager(repository, getMockExecutionsConfigProvider(), - getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockNodeExecutionRemoteURL) + nodeExecManager := NewNodeExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockNodeExecutionRemoteURL, nil) nodeExecution, err := nodeExecManager.GetNodeExecution(context.Background(), admin.NodeExecutionGetRequest{ Id: &nodeExecutionIdentifier, }) @@ -467,8 +461,7 @@ func TestGetNodeExecutionParentNode(t *testing.T) { }, }, nil }) - nodeExecManager := NewNodeExecutionManager(repository, getMockExecutionsConfigProvider(), - getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockNodeExecutionRemoteURL) + nodeExecManager := NewNodeExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockNodeExecutionRemoteURL, nil) nodeExecution, err := nodeExecManager.GetNodeExecution(context.Background(), admin.NodeExecutionGetRequest{ Id: &nodeExecutionIdentifier, }) @@ -489,8 +482,7 @@ func TestGetNodeExecution_DatabaseError(t *testing.T) { func(ctx context.Context, input interfaces.GetNodeExecutionInput) (models.NodeExecution, error) { return models.NodeExecution{}, expectedErr }) - nodeExecManager := NewNodeExecutionManager(repository, getMockExecutionsConfigProvider(), - getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockNodeExecutionRemoteURL) + nodeExecManager := NewNodeExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockNodeExecutionRemoteURL, nil) nodeExecution, err := nodeExecManager.GetNodeExecution(context.Background(), admin.NodeExecutionGetRequest{ Id: &nodeExecutionIdentifier, }) @@ -517,8 +509,7 @@ func TestGetNodeExecution_TransformerError(t *testing.T) { Closure: []byte("i'm invalid"), }, nil }) - nodeExecManager := NewNodeExecutionManager(repository, getMockExecutionsConfigProvider(), - getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockNodeExecutionRemoteURL) + nodeExecManager := NewNodeExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockNodeExecutionRemoteURL, nil) nodeExecution, err := nodeExecManager.GetNodeExecution(context.Background(), admin.NodeExecutionGetRequest{ Id: &nodeExecutionIdentifier, }) @@ -587,8 +578,7 @@ func TestListNodeExecutionsLevelZero(t *testing.T) { }, }, nil }) - nodeExecManager := NewNodeExecutionManager(repository, getMockExecutionsConfigProvider(), - getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockNodeExecutionRemoteURL) + nodeExecManager := NewNodeExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockNodeExecutionRemoteURL, nil) nodeExecutions, err := nodeExecManager.ListNodeExecutions(context.Background(), admin.NodeExecutionListRequest{ WorkflowExecutionId: &core.WorkflowExecutionIdentifier{ Project: "project", @@ -687,8 +677,7 @@ func TestListNodeExecutionsWithParent(t *testing.T) { }, }, nil }) - nodeExecManager := NewNodeExecutionManager(repository, getMockExecutionsConfigProvider(), - getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockNodeExecutionRemoteURL) + nodeExecManager := NewNodeExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockNodeExecutionRemoteURL, nil) nodeExecutions, err := nodeExecManager.ListNodeExecutions(context.Background(), admin.NodeExecutionListRequest{ WorkflowExecutionId: &core.WorkflowExecutionIdentifier{ Project: "project", @@ -722,8 +711,7 @@ func TestListNodeExecutionsWithParent(t *testing.T) { } func TestListNodeExecutions_InvalidParams(t *testing.T) { - nodeExecManager := NewNodeExecutionManager(nil, getMockExecutionsConfigProvider(), - getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockNodeExecutionRemoteURL) + nodeExecManager := NewNodeExecutionManager(nil, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockNodeExecutionRemoteURL, nil) _, err := nodeExecManager.ListNodeExecutions(context.Background(), admin.NodeExecutionListRequest{ Filters: "eq(execution.project, project)", }) @@ -749,8 +737,7 @@ func TestListNodeExecutions_DatabaseError(t *testing.T) { interfaces.NodeExecutionCollectionOutput, error) { return interfaces.NodeExecutionCollectionOutput{}, expectedErr }) - nodeExecManager := NewNodeExecutionManager(repository, getMockExecutionsConfigProvider(), - getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockNodeExecutionRemoteURL) + nodeExecManager := NewNodeExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockNodeExecutionRemoteURL, nil) nodeExecutions, err := nodeExecManager.ListNodeExecutions(context.Background(), admin.NodeExecutionListRequest{ WorkflowExecutionId: &core.WorkflowExecutionIdentifier{ Project: "project", @@ -788,8 +775,7 @@ func TestListNodeExecutions_TransformerError(t *testing.T) { }, }, nil }) - nodeExecManager := NewNodeExecutionManager(repository, getMockExecutionsConfigProvider(), - getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockNodeExecutionRemoteURL) + nodeExecManager := NewNodeExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockNodeExecutionRemoteURL, nil) nodeExecutions, err := nodeExecManager.ListNodeExecutions(context.Background(), admin.NodeExecutionListRequest{ WorkflowExecutionId: &core.WorkflowExecutionIdentifier{ Project: "project", @@ -817,8 +803,7 @@ func TestListNodeExecutions_NothingToReturn(t *testing.T) { listExecutionsCalled = true return interfaces.ExecutionCollectionOutput{}, nil }) - nodeExecManager := NewNodeExecutionManager(repository, getMockExecutionsConfigProvider(), - getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockNodeExecutionRemoteURL) + nodeExecManager := NewNodeExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockNodeExecutionRemoteURL, nil) _, err := nodeExecManager.ListNodeExecutions(context.Background(), admin.NodeExecutionListRequest{ WorkflowExecutionId: &core.WorkflowExecutionIdentifier{ Project: "project", @@ -915,8 +900,7 @@ func TestListNodeExecutionsForTask(t *testing.T) { }, }, nil }) - nodeExecManager := NewNodeExecutionManager(repository, getMockExecutionsConfigProvider(), - getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockNodeExecutionRemoteURL) + nodeExecManager := NewNodeExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockNodeExecutionRemoteURL, nil) nodeExecutions, err := nodeExecManager.ListNodeExecutionsForTask(context.Background(), admin.NodeExecutionForTaskListRequest{ TaskExecutionId: &core.TaskExecutionIdentifier{ NodeExecutionId: &core.NodeExecutionIdentifier{ @@ -1041,8 +1025,7 @@ func TestGetNodeExecutionData(t *testing.T) { } return fmt.Errorf("unexpected call to find value in storage [%v]", reference.String()) } - nodeExecManager := NewNodeExecutionManager(repository, getMockExecutionsConfigProvider(), mockStorage, - mockScope.NewTestScope(), mockNodeExecutionRemoteURL) + nodeExecManager := NewNodeExecutionManager(repository, getMockExecutionsConfigProvider(), mockStorage, mockScope.NewTestScope(), mockNodeExecutionRemoteURL, nil) dataResponse, err := nodeExecManager.GetNodeExecutionData(context.Background(), admin.NodeExecutionGetDataRequest{ Id: &nodeExecutionIdentifier, }) diff --git a/pkg/manager/impl/task_execution_manager.go b/pkg/manager/impl/task_execution_manager.go index ddb927e7c..4a44790fe 100644 --- a/pkg/manager/impl/task_execution_manager.go +++ b/pkg/manager/impl/task_execution_manager.go @@ -5,6 +5,9 @@ import ( "fmt" "strconv" + "github.com/golang/protobuf/proto" + notificationInterfaces "github.com/lyft/flyteadmin/pkg/async/notifications/interfaces" + "github.com/lyft/flytestdlib/storage" "github.com/lyft/flytestdlib/contextutils" @@ -40,14 +43,16 @@ type taskExecutionMetrics struct { ClosureSizeBytes prometheus.Summary TaskExecutionInputBytes prometheus.Summary TaskExecutionOutputBytes prometheus.Summary + PublishEventError prometheus.Counter } type TaskExecutionManager struct { - db repositories.RepositoryInterface - config runtimeInterfaces.Configuration - storageClient *storage.DataStore - metrics taskExecutionMetrics - urlData dataInterfaces.RemoteURLInterface + db repositories.RepositoryInterface + config runtimeInterfaces.Configuration + storageClient *storage.DataStore + metrics taskExecutionMetrics + urlData dataInterfaces.RemoteURLInterface + notificationClient notificationInterfaces.Publisher } func getTaskExecutionContext(ctx context.Context, identifier *core.TaskExecutionIdentifier) context.Context { @@ -173,6 +178,11 @@ func (m *TaskExecutionManager) CreateTaskExecutionEvent(ctx context.Context, req m.metrics.TaskExecutionsTerminated.Inc() } + if err = m.notificationClient.Publish(ctx, proto.MessageName(&request), &request); err != nil { + m.metrics.PublishEventError.Inc() + logger.Infof(ctx, "error publishing event [%+v] with err: [%v]", request.RequestId, err) + } + m.metrics.TaskExecutionEventsCreated.Inc() logger.Debugf(ctx, "Successfully recorded task execution event [%v]", request.Event) // TODO: we will want to return some scope information here soon! @@ -310,9 +320,7 @@ func (m *TaskExecutionManager) GetTaskExecutionData( return response, nil } -func NewTaskExecutionManager( - db repositories.RepositoryInterface, config runtimeInterfaces.Configuration, storageClient *storage.DataStore, - scope promutils.Scope, urlData dataInterfaces.RemoteURLInterface) interfaces.TaskExecutionInterface { +func NewTaskExecutionManager(db repositories.RepositoryInterface, config runtimeInterfaces.Configuration, storageClient *storage.DataStore, scope promutils.Scope, urlData dataInterfaces.RemoteURLInterface, publisher notificationInterfaces.Publisher) interfaces.TaskExecutionInterface { metrics := taskExecutionMetrics{ Scope: scope, ActiveTaskExecutions: scope.MustNewGauge("active_executions", @@ -333,12 +341,15 @@ func NewTaskExecutionManager( "size in bytes of serialized node execution inputs"), TaskExecutionOutputBytes: scope.MustNewSummary("output_size_bytes", "size in bytes of serialized node execution outputs"), + PublishEventError: scope.MustNewCounter("publish_event_error", + "overall count of publish event errors when invoking publish()"), } return &TaskExecutionManager{ - db: db, - config: config, - storageClient: storageClient, - metrics: metrics, - urlData: urlData, + db: db, + config: config, + storageClient: storageClient, + metrics: metrics, + urlData: urlData, + notificationClient: publisher, } } diff --git a/pkg/manager/impl/task_execution_manager_test.go b/pkg/manager/impl/task_execution_manager_test.go index 9cfb181b5..e7af57a34 100644 --- a/pkg/manager/impl/task_execution_manager_test.go +++ b/pkg/manager/impl/task_execution_manager_test.go @@ -175,8 +175,7 @@ func TestCreateTaskEvent(t *testing.T) { }, input) return nil }) - taskExecManager := NewTaskExecutionManager(repository, getMockExecutionsConfigProvider(), - getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockTaskExecutionRemoteURL) + taskExecManager := NewTaskExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockTaskExecutionRemoteURL, nil) resp, err := taskExecManager.CreateTaskExecutionEvent(context.Background(), taskEventRequest) assert.True(t, getTaskCalled) assert.True(t, createTaskCalled) @@ -289,8 +288,7 @@ func TestCreateTaskEvent_Update(t *testing.T) { OutputUri: expectedOutputResult.OutputUri, } - taskExecManager := NewTaskExecutionManager(repository, getMockExecutionsConfigProvider(), - getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockTaskExecutionRemoteURL) + taskExecManager := NewTaskExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockTaskExecutionRemoteURL, &mockPublisher) resp, err := taskExecManager.CreateTaskExecutionEvent(context.Background(), taskEventRequest) assert.True(t, getTaskCalled) assert.True(t, updateTaskCalled) @@ -305,8 +303,7 @@ func TestCreateTaskEvent_MissingExecution(t *testing.T) { func(ctx context.Context, input interfaces.GetNodeExecutionInput) (models.NodeExecution, error) { return models.NodeExecution{}, expectedErr }) - taskExecManager := NewTaskExecutionManager(repository, getMockExecutionsConfigProvider(), - getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockTaskExecutionRemoteURL) + taskExecManager := NewTaskExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockTaskExecutionRemoteURL, nil) resp, err := taskExecManager.CreateTaskExecutionEvent(context.Background(), taskEventRequest) assert.EqualError(t, err, "failed to get existing node execution id: [node_id:\"node-id\""+ " execution_id: ] "+ @@ -326,8 +323,7 @@ func TestCreateTaskEvent_CreateDatabaseError(t *testing.T) { func(ctx context.Context, input models.TaskExecution) error { return expectedErr }) - taskExecManager := NewTaskExecutionManager(repository, getMockExecutionsConfigProvider(), - getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockTaskExecutionRemoteURL) + taskExecManager := NewTaskExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockTaskExecutionRemoteURL, nil) resp, err := taskExecManager.CreateTaskExecutionEvent(context.Background(), taskEventRequest) assert.EqualError(t, err, expectedErr.Error()) assert.Nil(t, resp) @@ -367,8 +363,7 @@ func TestCreateTaskEvent_UpdateDatabaseError(t *testing.T) { func(ctx context.Context, execution models.TaskExecution) error { return expectedErr }) - nodeExecManager := NewTaskExecutionManager(repository, getMockExecutionsConfigProvider(), - getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockTaskExecutionRemoteURL) + nodeExecManager := NewTaskExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockTaskExecutionRemoteURL, nil) resp, err := nodeExecManager.CreateTaskExecutionEvent(context.Background(), taskEventRequest) assert.EqualError(t, err, expectedErr.Error()) assert.Nil(t, resp) @@ -401,8 +396,7 @@ func TestCreateTaskEvent_UpdateTerminalEventError(t *testing.T) { }, nil }) taskEventRequest.Event.Phase = core.TaskExecution_RUNNING - taskExecManager := NewTaskExecutionManager(repository, getMockExecutionsConfigProvider(), - getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockTaskExecutionRemoteURL) + taskExecManager := NewTaskExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockTaskExecutionRemoteURL, nil) resp, err := taskExecManager.CreateTaskExecutionEvent(context.Background(), taskEventRequest) assert.Nil(t, resp) @@ -469,8 +463,7 @@ func TestCreateTaskEvent_PhaseVersionChange(t *testing.T) { taskEventRequest.Event.PhaseVersion = uint32(1) taskEventRequest.Event.OccurredAt = taskEventUpdatedAtProto - taskExecManager := NewTaskExecutionManager(repository, getMockExecutionsConfigProvider(), - getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockTaskExecutionRemoteURL) + taskExecManager := NewTaskExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockTaskExecutionRemoteURL, &mockPublisher) resp, err := taskExecManager.CreateTaskExecutionEvent(context.Background(), taskEventRequest) assert.True(t, getTaskCalled) assert.True(t, updateTaskCalled) @@ -536,8 +529,7 @@ func TestGetTaskExecution(t *testing.T) { }, }, nil }) - taskExecManager := NewTaskExecutionManager(repository, getMockExecutionsConfigProvider(), - getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockTaskExecutionRemoteURL) + taskExecManager := NewTaskExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockTaskExecutionRemoteURL, nil) taskExecution, err := taskExecManager.GetTaskExecution(context.Background(), admin.TaskExecutionGetRequest{ Id: &core.TaskExecutionIdentifier{ TaskId: sampleTaskID, @@ -587,8 +579,7 @@ func TestGetTaskExecution_TransformerError(t *testing.T) { Closure: []byte("i'm an invalid task closure"), }, nil }) - taskExecManager := NewTaskExecutionManager(repository, getMockExecutionsConfigProvider(), - getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockTaskExecutionRemoteURL) + taskExecManager := NewTaskExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockTaskExecutionRemoteURL, nil) taskExecution, err := taskExecManager.GetTaskExecution(context.Background(), admin.TaskExecutionGetRequest{ Id: &core.TaskExecutionIdentifier{ TaskId: sampleTaskID, @@ -698,8 +689,7 @@ func TestListTaskExecutions(t *testing.T) { }, }, nil }) - taskExecManager := NewTaskExecutionManager(repository, getMockExecutionsConfigProvider(), - getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockTaskExecutionRemoteURL) + taskExecManager := NewTaskExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockTaskExecutionRemoteURL, nil) taskExecutions, err := taskExecManager.ListTaskExecutions(context.Background(), admin.TaskExecutionListRequest{ NodeExecutionId: &core.NodeExecutionIdentifier{ NodeId: "nodey b", @@ -770,8 +760,7 @@ func TestListTaskExecutions_NoFilters(t *testing.T) { listTaskCalled = true return interfaces.TaskExecutionCollectionOutput{}, nil }) - taskExecManager := NewTaskExecutionManager(repository, getMockExecutionsConfigProvider(), - getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockTaskExecutionRemoteURL) + taskExecManager := NewTaskExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockTaskExecutionRemoteURL, nil) _, err := taskExecManager.ListTaskExecutions(context.Background(), admin.TaskExecutionListRequest{ Token: "1", Limit: 99, @@ -789,8 +778,7 @@ func TestListTaskExecutions_NoLimit(t *testing.T) { getTaskCalled = true return interfaces.TaskExecutionCollectionOutput{}, nil }) - taskExecManager := NewTaskExecutionManager(repository, getMockExecutionsConfigProvider(), - getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockTaskExecutionRemoteURL) + taskExecManager := NewTaskExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockTaskExecutionRemoteURL, nil) _, err := taskExecManager.ListTaskExecutions(context.Background(), admin.TaskExecutionListRequest{ Limit: 0, }) @@ -821,8 +809,7 @@ func TestListTaskExecutions_NothingToReturn(t *testing.T) { listTasksCalled = true return interfaces.TaskCollectionOutput{}, nil }) - taskExecManager := NewTaskExecutionManager(repository, getMockExecutionsConfigProvider(), - getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockTaskExecutionRemoteURL) + taskExecManager := NewTaskExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockTaskExecutionRemoteURL, nil) _, err := taskExecManager.ListTaskExecutions(context.Background(), admin.TaskExecutionListRequest{ NodeExecutionId: &core.NodeExecutionIdentifier{ ExecutionId: &core.WorkflowExecutionIdentifier{ @@ -924,8 +911,7 @@ func TestGetTaskExecutionData(t *testing.T) { } return fmt.Errorf("unexpected call to find value in storage [%v]", reference.String()) } - taskExecManager := NewTaskExecutionManager(repository, getMockExecutionsConfigProvider(), mockStorage, - mockScope.NewTestScope(), mockTaskExecutionRemoteURL) + taskExecManager := NewTaskExecutionManager(repository, getMockExecutionsConfigProvider(), mockStorage, mockScope.NewTestScope(), mockTaskExecutionRemoteURL, nil) dataResponse, err := taskExecManager.GetTaskExecutionData(context.Background(), admin.TaskExecutionGetDataRequest{ Id: &core.TaskExecutionIdentifier{ TaskId: sampleTaskID, diff --git a/pkg/rpc/adminservice/base.go b/pkg/rpc/adminservice/base.go index 376233868..806970fa4 100644 --- a/pkg/rpc/adminservice/base.go +++ b/pkg/rpc/adminservice/base.go @@ -99,6 +99,7 @@ func NewAdminServer(kubeConfig, master string) *AdminService { publisher := notifications.NewNotificationsPublisher(*configuration.ApplicationConfiguration().GetNotificationsConfig(), adminScope) processor := notifications.NewNotificationsProcessor(*configuration.ApplicationConfiguration().GetNotificationsConfig(), adminScope) + eventPublisher := notifications.NewEventsPublisher(*configuration.ApplicationConfiguration().GetExternalEventsConfig(), adminScope) go func() { logger.Info(context.Background(), "Started processing notifications.") processor.StartProcessing() @@ -131,10 +132,7 @@ func NewAdminServer(kubeConfig, master string) *AdminService { db, configuration, workflowengine.NewCompiler(), dataStorageClient, applicationConfiguration.MetadataStoragePrefix, adminScope.NewSubScope("workflow_manager")) namedEntityManager := manager.NewNamedEntityManager(db, configuration, adminScope.NewSubScope("named_entity_manager")) - executionManager := manager.NewExecutionManager( - db, configuration, dataStorageClient, workflowExecutor, adminScope.NewSubScope("execution_manager"), - adminScope.NewSubScope("user_execution_metrics"), publisher, urlData, workflowManager, - namedEntityManager) + executionManager := manager.NewExecutionManager(db, configuration, dataStorageClient, workflowExecutor, adminScope.NewSubScope("execution_manager"), adminScope.NewSubScope("user_execution_metrics"), publisher, urlData, workflowManager, namedEntityManager, eventPublisher) scheduledWorkflowExecutor := workflowScheduler.GetWorkflowExecutor(executionManager, launchPlanManager) logger.Info(context.Background(), "Successfully initialized a new scheduled workflow executor") @@ -160,10 +158,10 @@ func NewAdminServer(kubeConfig, master string) *AdminService { LaunchPlanManager: launchPlanManager, ExecutionManager: executionManager, NamedEntityManager: namedEntityManager, - NodeExecutionManager: manager.NewNodeExecutionManager( - db, configuration, dataStorageClient, adminScope.NewSubScope("node_execution_manager"), urlData), - TaskExecutionManager: manager.NewTaskExecutionManager( - db, configuration, dataStorageClient, adminScope.NewSubScope("task_execution_manager"), urlData), + NodeExecutionManager: manager.NewNodeExecutionManager(db, configuration, dataStorageClient, + adminScope.NewSubScope("node_execution_manager"), urlData, eventPublisher), + TaskExecutionManager: manager.NewTaskExecutionManager(db, configuration, dataStorageClient, + adminScope.NewSubScope("task_execution_manager"), urlData, eventPublisher), ProjectManager: manager.NewProjectManager(db, configuration), ResourceManager: resources.NewResourceManager(db, configuration.ApplicationConfiguration()), Metrics: InitMetrics(adminScope), diff --git a/pkg/runtime/application_config_provider.go b/pkg/runtime/application_config_provider.go index 58032b67e..1ad15c8ef 100644 --- a/pkg/runtime/application_config_provider.go +++ b/pkg/runtime/application_config_provider.go @@ -16,6 +16,7 @@ const scheduler = "scheduler" const remoteData = "remoteData" const notifications = "notifications" const domains = "domains" +const externalEvents = "externalEvents" var databaseConfig = config.MustRegisterSection(database, &interfaces.DbConfigSection{}) var flyteAdminConfig = config.MustRegisterSection(flyteAdmin, &interfaces.ApplicationConfig{}) @@ -23,6 +24,7 @@ var schedulerConfig = config.MustRegisterSection(scheduler, &interfaces.Schedule var remoteDataConfig = config.MustRegisterSection(remoteData, &interfaces.RemoteDataConfig{}) var notificationsConfig = config.MustRegisterSection(notifications, &interfaces.NotificationsConfig{}) var domainsConfig = config.MustRegisterSection(domains, &interfaces.DomainsConfig{}) +var externalEventsConfig = config.MustRegisterSection(externalEvents, &interfaces.ExternalEventsConfig{}) // Implementation of an interfaces.ApplicationConfiguration type ApplicationConfigurationProvider struct{} @@ -72,6 +74,11 @@ func (p *ApplicationConfigurationProvider) GetNotificationsConfig() *interfaces. func (p *ApplicationConfigurationProvider) GetDomainsConfig() *interfaces.DomainsConfig { return domainsConfig.GetConfig().(*interfaces.DomainsConfig) } + +func (p *ApplicationConfigurationProvider) GetExternalEventsConfig() *interfaces.ExternalEventsConfig { + return externalEventsConfig.GetConfig().(*interfaces.ExternalEventsConfig) +} + func NewApplicationConfigurationProvider() interfaces.ApplicationConfiguration { return &ApplicationConfigurationProvider{} } diff --git a/pkg/runtime/interfaces/application_configuration.go b/pkg/runtime/interfaces/application_configuration.go index d00878522..6cffceb29 100644 --- a/pkg/runtime/interfaces/application_configuration.go +++ b/pkg/runtime/interfaces/application_configuration.go @@ -146,6 +146,29 @@ type NotificationsEmailerConfig struct { Body string `json:"body"` } +// This section handles configuration for the workflow notifications pipeline. +type EventsPublisherConfig struct { + // The topic which events should be published, e.g. node, task, workflow + TopicName string `json:"topicName"` + // Event types: task, node, workflow executions + EventTypes []string `json:"eventTypes"` +} + +type ExternalEventsConfig struct { + Enable bool `json:"enable"` + // Defines the cloud provider that backs the scheduler. In the absence of a specification the no-op, 'local' + // scheme is used. + Type string `json:"type"` + AWSConfig AWSConfig `json:"aws"` + GCPConfig GCPConfig `json:"gcp"` + // Publish events to a pubsub tops + EventsPublisherConfig EventsPublisherConfig `json:"eventsPublisher"` + // Number of times to attempt recreating a notifications processor client should there be any disruptions. + ReconnectAttempts int `json:"reconnectAttempts"` + // Specifies the time interval to wait before attempting to reconnect the notifications processor client. + ReconnectDelaySeconds int `json:"reconnectDelaySeconds"` +} + // Configuration specific to notifications handling type NotificationsConfig struct { // Defines the cloud provider that backs the scheduler. In the absence of a specification the no-op, 'local' @@ -182,4 +205,5 @@ type ApplicationConfiguration interface { GetRemoteDataConfig() *RemoteDataConfig GetNotificationsConfig() *NotificationsConfig GetDomainsConfig() *DomainsConfig + GetExternalEventsConfig() *ExternalEventsConfig } diff --git a/pkg/runtime/mocks/mock_application_provider.go b/pkg/runtime/mocks/mock_application_provider.go index 0d9973dbc..33ac9a9b9 100644 --- a/pkg/runtime/mocks/mock_application_provider.go +++ b/pkg/runtime/mocks/mock_application_provider.go @@ -5,12 +5,13 @@ import ( ) type MockApplicationProvider struct { - dbConfig interfaces.DbConfig - topLevelConfig interfaces.ApplicationConfig - schedulerConfig interfaces.SchedulerConfig - remoteDataConfig interfaces.RemoteDataConfig - notificationsConfig interfaces.NotificationsConfig - domainsConfig interfaces.DomainsConfig + dbConfig interfaces.DbConfig + topLevelConfig interfaces.ApplicationConfig + schedulerConfig interfaces.SchedulerConfig + remoteDataConfig interfaces.RemoteDataConfig + notificationsConfig interfaces.NotificationsConfig + domainsConfig interfaces.DomainsConfig + externalEventsConfig interfaces.ExternalEventsConfig } func (p *MockApplicationProvider) GetDbConfig() interfaces.DbConfig { @@ -60,3 +61,11 @@ func (p *MockApplicationProvider) GetDomainsConfig() *interfaces.DomainsConfig { func (p *MockApplicationProvider) SetDomainsConfig(domainsConfig interfaces.DomainsConfig) { p.domainsConfig = domainsConfig } + +func (p *MockApplicationProvider) SetExternalEventsConfig(externalEventsConfig interfaces.ExternalEventsConfig) { + p.externalEventsConfig = externalEventsConfig +} + +func (p *MockApplicationProvider) GetExternalEventsConfig() *interfaces.ExternalEventsConfig { + return &p.externalEventsConfig +} From ee98c708a12d57a4fea40ee937abb325fd68e06b Mon Sep 17 00:00:00 2001 From: Katrina Rogan Date: Mon, 1 Mar 2021 09:56:11 -0800 Subject: [PATCH 20/22] Pull in array job idl changes (#152) --- go.mod | 2 +- go.sum | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 8f0419ca4..fce4ca6f9 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/jinzhu/gorm v1.9.12 github.com/kelseyhightower/envconfig v1.4.0 // indirect github.com/lib/pq v1.3.0 - github.com/lyft/flyteidl v0.18.11 + github.com/lyft/flyteidl v0.18.14 github.com/lyft/flytepropeller v0.5.14 github.com/lyft/flytestdlib v0.3.9 github.com/magiconair/properties v1.8.1 diff --git a/go.sum b/go.sum index 4ee515ec9..5d22baedf 100644 --- a/go.sum +++ b/go.sum @@ -457,6 +457,10 @@ github.com/lyft/flyteidl v0.17.0/go.mod h1:/zQXxuHO11u/saxTTZc8oYExIGEShXB+xCB1/ github.com/lyft/flyteidl v0.18.9/go.mod h1:/zQXxuHO11u/saxTTZc8oYExIGEShXB+xCB1/F1Cu20= github.com/lyft/flyteidl v0.18.11 h1:24NaFYWxANhRbwKfvkgu8axGTWUcl1tgZBqNJutKNJ8= github.com/lyft/flyteidl v0.18.11/go.mod h1:/zQXxuHO11u/saxTTZc8oYExIGEShXB+xCB1/F1Cu20= +github.com/lyft/flyteidl v0.18.14-0.20210224014048-73cf22d60fd5 h1:m2CBZEr1RC69bZe8VG3xryNfe+jOBUTKze9Zj6sJCms= +github.com/lyft/flyteidl v0.18.14-0.20210224014048-73cf22d60fd5/go.mod h1:JTJC2VqrpEWM/76lPF2Dj9l4FA8FjZh67U0RYuq/Aes= +github.com/lyft/flyteidl v0.18.14 h1:c5tBItxLZA77HNRdYiHEZ9UyVUq9I3wiYKDuF/Om/xs= +github.com/lyft/flyteidl v0.18.14/go.mod h1:JTJC2VqrpEWM/76lPF2Dj9l4FA8FjZh67U0RYuq/Aes= github.com/lyft/flyteplugins v0.5.20/go.mod h1:1G2YORr5JZhfl4dyHGIO4/+ShNMul5DPnoBEdpJfuNc= github.com/lyft/flytepropeller v0.5.14 h1:6S93Q1tTnoc5iFKe4Sw4qZzQHZgfuTQd6zUdxaZGAik= github.com/lyft/flytepropeller v0.5.14/go.mod h1:P63eiQpUI/57ifXNoHc/NSszU8PYLcJ+fujI4wQ99rI= From 194804761b74ce0c1ce67c5c3421746abe083525 Mon Sep 17 00:00:00 2001 From: Haytham Abuelfutuh Date: Mon, 1 Mar 2021 20:12:46 -0800 Subject: [PATCH 21/22] Support Google OAuth2 and other OIdC providers (#147) * Move scopes to config Signed-off-by: Haytham Abuelfutuh * missed Signed-off-by: Haytham Abuelfutuh * wip Signed-off-by: Haytham Abuelfutuh * Update handlers.go Signed-off-by: Haytham Abuelfutuh * go get propeller at v0.5.12 (#146) Signed-off-by: Haytham Abuelfutuh * Update to using latest flytepropeller v0.5.13 (#148) Signed-off-by: Haytham Abuelfutuh * Update propeller to 0.5.14 (#149) * Update propeller * cleanup Signed-off-by: Haytham Abuelfutuh * Filter executions by user (#150) Signed-off-by: Haytham Abuelfutuh * Update CI post migration Signed-off-by: Haytham Abuelfutuh * Update codecov link Signed-off-by: Haytham Abuelfutuh * Publish raw events (#151) Signed-off-by: Haytham Abuelfutuh * fix test Signed-off-by: Haytham Abuelfutuh * wip Signed-off-by: Haytham Abuelfutuh * Fix token retrieval from cookies Signed-off-by: Haytham Abuelfutuh * Fix unit tests and lint Signed-off-by: Haytham Abuelfutuh * Move to const Signed-off-by: Haytham Abuelfutuh * Revert Auth config Signed-off-by: Haytham Abuelfutuh * Revert kube config path Signed-off-by: Haytham Abuelfutuh * Use access token when posting to IdP Signed-off-by: Haytham Abuelfutuh * Ignore refresh token error Signed-off-by: Haytham Abuelfutuh * Fix tests Signed-off-by: Haytham Abuelfutuh * Expose openId metadata endpoint and expose scopes in /config endpoint Signed-off-by: Haytham Abuelfutuh * mod tidy Signed-off-by: Haytham Abuelfutuh * rename Signed-off-by: Haytham Abuelfutuh * lint Signed-off-by: Haytham Abuelfutuh * unit tests Signed-off-by: Haytham Abuelfutuh Co-authored-by: Yee Hing Tong Co-authored-by: Katrina Rogan Co-authored-by: tnsetting <38207208+tnsetting@users.noreply.github.com> --- cmd/entrypoints/serve.go | 6 +- flyteadmin_config.yaml | 11 ++-- go.sum | 64 ------------------- pkg/auth/auth_context.go | 56 ++++++++++------ pkg/auth/config/config.go | 5 ++ pkg/auth/constants.go | 9 ++- pkg/auth/cookie.go | 36 ++++++++++- pkg/auth/cookie_manager.go | 60 ++++++++++------- pkg/auth/cookie_manager_test.go | 29 ++++++--- pkg/auth/handlers.go | 50 ++++++++------- pkg/auth/handlers_test.go | 19 ++++-- pkg/auth/interfaces/context.go | 3 +- pkg/auth/interfaces/cookie.go | 2 +- .../mocks/authentication_context.go | 56 ++++++++++++---- pkg/auth/token.go | 11 +++- pkg/config/config.go | 6 ++ pkg/rpc/config/flyte_client.go | 6 +- pkg/rpc/config/flyte_client_test.go | 1 + 18 files changed, 257 insertions(+), 173 deletions(-) diff --git a/cmd/entrypoints/serve.go b/cmd/entrypoints/serve.go index c36d523b2..1325e95c3 100644 --- a/cmd/entrypoints/serve.go +++ b/cmd/entrypoints/serve.go @@ -153,8 +153,12 @@ func newHTTPServer(ctx context.Context, cfg *config.ServerConfig, authContext in if authContext.GetUserInfoURL() != nil && authContext.GetUserInfoURL().String() != "" { mux.HandleFunc("/me", auth.GetMeEndpointHandler(ctx, authContext)) } + + // The metadata endpoint is an RFC-defined constant, but we need a leading / for the handler to pattern match correctly. + mux.HandleFunc(fmt.Sprintf("/%s", auth.OIdCMetadataEndpoint), auth.GetOIdCMetadataEndpointRedirectHandler(ctx, authContext)) + // The metadata endpoint is an RFC-defined constant, but we need a leading / for the handler to pattern match correctly. - mux.HandleFunc(fmt.Sprintf("/%s", auth.MetadataEndpoint), auth.GetMetadataEndpointRedirectHandler(ctx, authContext)) + mux.HandleFunc(fmt.Sprintf("/%s", auth.OAuth2MetadataEndpoint), auth.GetOAuth2MetadataEndpointRedirectHandler(ctx, authContext)) // This option translates HTTP authorization data (cookies) into a gRPC metadata field gwmuxOptions = append(gwmuxOptions, runtime.WithMetadata(auth.GetHTTPRequestCookieToMetadataHandler(authContext))) diff --git a/flyteadmin_config.yaml b/flyteadmin_config.yaml index e62521ada..0b55257d8 100644 --- a/flyteadmin_config.yaml +++ b/flyteadmin_config.yaml @@ -8,10 +8,13 @@ server: grpcServerReflection: true security: secure: false - ssl: - certificateFile: "/path/to/server.pem" - keyFile: "/path/to/server.key" useAuth: false + allowCors: true + allowedOrigins: + # Accepting all domains for Sandbox installation + - "*" + allowedHeaders: + - "Content-Type" oauth: clientId: yourclientid clientSecretFile: "/path/to/oauth/secret" @@ -73,7 +76,7 @@ notifications: accountId: "bar" emailer: subject: "Notice: Execution \"{{ name }}\" has {{ phase }} in \"{{ domain }}\"." - sender: "flyte-notifications@example.com" + sender: "flyte-notifications@example.com" body: > Execution \"{{ name }}\" has {{ phase }} in \"{{ domain }}\". View details at diff --git a/go.sum b/go.sum index 5d22baedf..6137e385d 100644 --- a/go.sum +++ b/go.sum @@ -455,16 +455,11 @@ github.com/lyft/apimachinery v0.0.0-20191031200210-047e3ea32d7f/go.mod h1:llRdnz github.com/lyft/datacatalog v0.2.1/go.mod h1:ktrPvzTDUwHO5Lv0hLH38zLHnOJ++rGoAO0iQ/sIPJ4= github.com/lyft/flyteidl v0.17.0/go.mod h1:/zQXxuHO11u/saxTTZc8oYExIGEShXB+xCB1/F1Cu20= github.com/lyft/flyteidl v0.18.9/go.mod h1:/zQXxuHO11u/saxTTZc8oYExIGEShXB+xCB1/F1Cu20= -github.com/lyft/flyteidl v0.18.11 h1:24NaFYWxANhRbwKfvkgu8axGTWUcl1tgZBqNJutKNJ8= -github.com/lyft/flyteidl v0.18.11/go.mod h1:/zQXxuHO11u/saxTTZc8oYExIGEShXB+xCB1/F1Cu20= -github.com/lyft/flyteidl v0.18.14-0.20210224014048-73cf22d60fd5 h1:m2CBZEr1RC69bZe8VG3xryNfe+jOBUTKze9Zj6sJCms= -github.com/lyft/flyteidl v0.18.14-0.20210224014048-73cf22d60fd5/go.mod h1:JTJC2VqrpEWM/76lPF2Dj9l4FA8FjZh67U0RYuq/Aes= github.com/lyft/flyteidl v0.18.14 h1:c5tBItxLZA77HNRdYiHEZ9UyVUq9I3wiYKDuF/Om/xs= github.com/lyft/flyteidl v0.18.14/go.mod h1:JTJC2VqrpEWM/76lPF2Dj9l4FA8FjZh67U0RYuq/Aes= github.com/lyft/flyteplugins v0.5.20/go.mod h1:1G2YORr5JZhfl4dyHGIO4/+ShNMul5DPnoBEdpJfuNc= github.com/lyft/flytepropeller v0.5.14 h1:6S93Q1tTnoc5iFKe4Sw4qZzQHZgfuTQd6zUdxaZGAik= github.com/lyft/flytepropeller v0.5.14/go.mod h1:P63eiQpUI/57ifXNoHc/NSszU8PYLcJ+fujI4wQ99rI= -github.com/lyft/flytestdlib v0.3.0 h1:nIkX4MlyYdcLLzaF35RI2P5BhARt+qMgHoFto8eVNzU= github.com/lyft/flytestdlib v0.3.0/go.mod h1:LJPPJlkFj+wwVWMrQT3K5JZgNhZi2mULsCG4ZYhinhU= github.com/lyft/flytestdlib v0.3.9 h1:NaKp9xkeWWwhVvqTOcR/FqlASy1N2gu/kN7PVe4S7YI= github.com/lyft/flytestdlib v0.3.9/go.mod h1:LJPPJlkFj+wwVWMrQT3K5JZgNhZi2mULsCG4ZYhinhU= @@ -479,13 +474,11 @@ github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.6 h1:6Su7aK7lXmJ/U79bYtBjLNaha4Fs1Rg9plHpcH+vvnE= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.11 h1:FxPOTFNqGkuDUGi3H/qkUbQO4ZiBa2brKq5r0l8TGeM= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= @@ -496,7 +489,6 @@ github.com/mattn/go-sqlite3 v2.0.1+incompatible h1:xQ15muvnzGBHpIpdrNi1DA5x0+TcB github.com/mattn/go-sqlite3 v2.0.1+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= @@ -511,7 +503,6 @@ github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8m github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= -github.com/ncw/swift v1.0.49 h1:eQaKIjSt/PXLKfYgzg01nevmO+CMXfXGRhB1gOhDs7E= github.com/ncw/swift v1.0.49/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= github.com/ncw/swift v1.0.50 h1:E01b5bVIssNhx2KnzAjMWEXkKrb8ytTqCDWY7lqmWjA= github.com/ncw/swift v1.0.50/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= @@ -523,7 +514,6 @@ github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:v github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w= github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.11.0 h1:JAKSXpt1YjtLA7YpPiqO9ss6sNXEsPfSGdwN0UHqzrw= @@ -531,7 +521,6 @@ github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+ github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.8.1 h1:C5Dqfs/LeauYDX0jJXIe2SWmwCbGzx9yF8C8xy3Lh34= @@ -547,7 +536,6 @@ github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR github.com/pierrec/lz4 v0.0.0-20190327172049-315a67e90e41/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -565,18 +553,14 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= -github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.3.0 h1:miYCvYqFXtl/J9FIy8eNpBfYthAEFg+Ys0XyUVEcDsc= github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= github.com/prometheus/client_golang v1.5.0 h1:Ctq0iGpCmr3jeP77kbF2UxgvRwzWWz+4Bh9/vJTyg1A= github.com/prometheus/client_golang v1.5.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.1.0 h1:ElTg5tNp4DqfV7UQjDqv2+RJlNzsDtvNAWccbItceIE= github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -586,7 +570,6 @@ github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7q github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.7.0 h1:L+1lyG48J1zAQXA3RBX/nG/B3gjlHq0zTt2tlbJLyCY= github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/common v0.9.1 h1:KOMtN28tlbam3/7ZKEYKHhKoJZYYj3gMH4uc62x7X7U= github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= @@ -596,7 +579,6 @@ github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.0.10 h1:QJQN3jYQhkamO4mhfUWqdDH2asK7ONOI9MTWjyAxNKM= github.com/prometheus/procfs v0.0.10/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= @@ -610,7 +592,6 @@ github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= -github.com/satori/uuid v1.2.0 h1:6TFY4nxn5XwBx0gDfzbEMCNT6k4N/4FNIuN8RACZ0KI= github.com/satori/uuid v1.2.0/go.mod h1:B8HLsPLik/YNn6KKWVMDJ8nzCL8RP5WyfsnmvnAEwIU= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= @@ -628,12 +609,10 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasO github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v0.0.6 h1:breEStsVwemnKh2/s6gMvSdMEkwW0sK8vGStnlVBMCs= github.com/spf13/cobra v0.0.6/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= @@ -646,22 +625,18 @@ github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.3.2 h1:VUFqw5KcqRf7i70GOzW7N+Q7+gxVBkSSqiXB12+JQ4M= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/spf13/viper v1.6.2 h1:7aKfF+e8/k68gda3LOjo5RxiUqddoFxVq4BKBPrxk5E= github.com/spf13/viper v1.6.2/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -679,7 +654,6 @@ github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= @@ -690,7 +664,6 @@ go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qL go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2 h1:75k/FF0Q2YM8QYo07VPddOLBslDt1MZOdEslOHvmzAs= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -720,9 +693,7 @@ golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200117160349-530e935923ad h1:Jh8cai0fqIK+f6nG0UgPW5wFk8wmiMhM3AyciDBdtQg= golang.org/x/crypto v0.0.0-20200117160349-530e935923ad/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59 h1:3zb4D3T4G8jdExgVU/95+vQXfpEPiMdCaZgmGVxjNHM= golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -757,7 +728,6 @@ golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKG golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -789,24 +759,17 @@ golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa h1:F+8P+gmewFQYRk6JoLQLwjBCTu3mcIURZfNkVweuRKA= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a h1:GuSPYbZzB5/dcLNCwLQLsg3obCJtX9IJhpXkvY7kzk0= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344 h1:vGXIOMxbNfDTk/aXCmfdLgkrSV+Z2tcbze+pEc3v5W4= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= @@ -816,11 +779,8 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 h1:qwRHBd0NqMbJxfbotnDhm2ByMI1Shq4Y6oRJo21SGJA= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -856,18 +816,13 @@ golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9 h1:1/DFK4b7JH8DmkqhUk48onnSfrPzImPoVxuomtbT2nk= golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4 h1:sfkvUWPNGwSV+8/fNqctR5lS2AqCSqYwXdrjCxp/dXo= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527 h1:uYVVQ9WP/Ds2ROhcaGPeIdVq0RIXVLwsHlnvJ+cT1So= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200327173247-9dae0f8f5775 h1:TC0v2RSO1u2kn1ZugjrFXkRZAEaqMN/RW+OTZkBzmLE= golang.org/x/sys v0.0.0-20200327173247-9dae0f8f5775/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d h1:nc5K6ox/4lTFbMVSL9WRR81ixkcwXThoiF6yf+R9scA= golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 h1:myAQVi0cGEoqQVR5POX+8RR2mrocKqNN1hmeMqhX27k= @@ -876,7 +831,6 @@ golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fq golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -925,16 +879,12 @@ golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb h1:iKlO7ROJc6SttHKlxzwGytRtBUqX4VARrNTgP2YLX5M= golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200818005847-188abfa75333 h1:a6ryybeZHQf5qnBc6IwRfVnI/75UmdtJo71f0//8Dqo= -golang.org/x/tools v0.0.0-20200818005847-188abfa75333/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -954,7 +904,6 @@ google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsb google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.16.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.17.0 h1:0q95w+VuFtv4PAx4PZVQdBMmYbaCHbnfKaEiDIcVyag= google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.20.0 h1:jz2KixHX7EcCPiQrySzPdnYT7DbINAypCqKZ1Z7GM40= @@ -982,15 +931,12 @@ google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvx google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150 h1:VPpdpQkGvFicX9yo4G5oxZPi9ALBnEOZblPSa/Wa2m4= google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= google.golang.org/genproto v0.0.0-20200205142000-a86caf926a67/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce h1:1mbrb1tUU+Zmt5C94IGKADBTJZjZXAd+BubWi7r9EiI= google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200312145019-da6875a35672 h1:jiDSspVssiikoRPFHT6pYrL+CL6/yIc3b9AuHO/4xik= google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940 h1:MRHtG0U6SnaUb+s+LhNE1qt1FQ1wlhqr5E4usBKC0uA= google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= @@ -1002,21 +948,16 @@ google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij google.golang.org/grpc v1.22.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.24.0 h1:vb/1TCsVn3DcJlQ0Gs1yB1pKI6Do2/QNwxdKqmc/b0s= google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1 h1:zvIju4sqAGvwKspUQOhwnpcqSbzi7/H6QomNNjTL4sk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.28.0 h1:bO/TA4OxCOummhSf10siHuG7vJOiwh7SpRpFZDkOgl4= google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -1030,7 +971,6 @@ gopkg.in/inf.v0 v0.9.0/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.51.1 h1:GyboHr4UqMiLUybYjd22ZjQIKEJEpgtLXtuGbR21Oho= gopkg.in/ini.v1 v1.51.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.54.0 h1:oM5ElzbIi7gwLnNbPX2M25ED1vSAK3B6dex50eS/6Fs= gopkg.in/ini.v1 v1.54.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= @@ -1065,7 +1005,6 @@ honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3 h1:sXmLre5bzIR6ypkjXCDI3jHPssRhc8KD/Ome589sc3U= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= @@ -1081,7 +1020,6 @@ k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8 k8s.io/gengo v0.0.0-20190822140433-26a664648505/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= -k8s.io/klog v0.4.0 h1:lCJCxf/LIowc2IGS9TPjWDyXY4nOmdGdfcwwDQCOURQ= k8s.io/klog v0.4.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= @@ -1093,7 +1031,6 @@ k8s.io/kube-openapi v0.0.0-20200204173128-addea2498afe/go.mod h1:GRQhZsXIAJ1xR0C k8s.io/utils v0.0.0-20190506122338-8fab8cb257d5/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= k8s.io/utils v0.0.0-20190801114015-581e00157fb1/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= -k8s.io/utils v0.0.0-20200124190032-861946025e34 h1:HjlUD6M0K3P8nRXmr2B9o4F9dUy9TCj/aEpReeyi6+k= k8s.io/utils v0.0.0-20200124190032-861946025e34/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= k8s.io/utils v0.0.0-20200229041039-0a110f9eb7ab h1:I3f2hcBrepGRXI1z4sukzAb8w1R4eqbsHrAsx06LGYM= k8s.io/utils v0.0.0-20200229041039-0a110f9eb7ab/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= @@ -1112,7 +1049,6 @@ sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:w sigs.k8s.io/structured-merge-diff v1.0.1-0.20191108220359-b1b620dd3f06/go.mod h1:/ULNhyfzRopfcjskuui0cTITekDduZ7ycKN3oUT9R18= sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= sigs.k8s.io/testing_frameworks v0.1.1/go.mod h1:VVBKrHmJ6Ekkfz284YKhQePcdycOzNH9qL6ht1zEr/U= -sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/pkg/auth/auth_context.go b/pkg/auth/auth_context.go index b6f757480..95bc2c307 100644 --- a/pkg/auth/auth_context.go +++ b/pkg/auth/auth_context.go @@ -22,15 +22,16 @@ const ( // Please see the comment on the corresponding AuthenticationContext for more information. type Context struct { - oauth2 *oauth2.Config - claims config.Claims - cookieManager interfaces.CookieHandler - oidcProvider *oidc.Provider - options config.OAuthOptions - userInfoURL *url.URL - baseURL *url.URL - metadataURL *url.URL - httpClient *http.Client + oauth2 *oauth2.Config + claims config.Claims + cookieManager interfaces.CookieHandler + oidcProvider *oidc.Provider + options config.OAuthOptions + userInfoURL *url.URL + baseURL *url.URL + oauth2MetadataURL *url.URL + oidcMetadataURL *url.URL + httpClient *http.Client } func (c Context) OAuth2Config() *oauth2.Config { @@ -65,8 +66,12 @@ func (c Context) GetBaseURL() *url.URL { return c.baseURL } -func (c Context) GetMetadataURL() *url.URL { - return c.metadataURL +func (c Context) GetOAuth2MetadataURL() *url.URL { + return c.oauth2MetadataURL +} + +func (c Context) GetOIdCMetadataURL() *url.URL { + return c.oidcMetadataURL } const ( @@ -85,6 +90,7 @@ func NewAuthenticationContext(ctx context.Context, options config.OAuthOptions) if err != nil { return Context{}, errors.Wrapf(ErrAuthContext, err, "Error creating OAuth2 library configuration") } + result.oauth2 = &oauth2Config // Construct the cookie manager object. @@ -92,23 +98,27 @@ func NewAuthenticationContext(ctx context.Context, options config.OAuthOptions) if err != nil { return Context{}, errors.Wrapf(ErrConfigFileRead, err, "Could not read hash key file") } + blockKeyBytes, err := ioutil.ReadFile(options.CookieBlockKeyFile) if err != nil { return Context{}, errors.Wrapf(ErrConfigFileRead, err, "Could not read block key file") } + cookieManager, err := NewCookieManager(ctx, string(hashKeyBytes), string(blockKeyBytes)) if err != nil { logger.Errorf(ctx, "Error creating cookie manager %s", err) return Context{}, errors.Wrapf(ErrAuthContext, err, "Error creating cookie manager") } + result.cookieManager = cookieManager // Construct an oidc Provider, which needs its own http Client. oidcCtx := oidc.ClientContext(ctx, &http.Client{}) provider, err := oidc.NewProvider(oidcCtx, options.Claims.Issuer) if err != nil { - return Context{}, errors.Wrapf(ErrAuthContext, err, "Error creating oidc provider") + return Context{}, errors.Wrapf(ErrAuthContext, err, "Error creating oidc provider w/ issuer [%v]", options.Claims.Issuer) } + result.oidcProvider = provider // TODO: Convert all the URLs in this config to the config.URL type @@ -120,16 +130,25 @@ func NewAuthenticationContext(ctx context.Context, options config.OAuthOptions) logger.Errorf(ctx, "Error parsing base URL %s", err) return Context{}, errors.Wrapf(ErrAuthContext, err, "Error parsing IDP base URL") } + logger.Infof(ctx, "Base IDP URL is %s", base) result.baseURL = base - metadataURL, err := url.Parse(MetadataEndpoint) + result.oauth2MetadataURL, err = url.Parse(OAuth2MetadataEndpoint) if err != nil { - logger.Errorf(ctx, "Error parsing metadata URL %s", err) + logger.Errorf(ctx, "Error parsing oauth2 metadata URL %s", err) return Context{}, errors.Wrapf(ErrAuthContext, err, "Error parsing metadata URL") } - logger.Infof(ctx, "Metadata endpoint is %s", metadataURL) - result.metadataURL = metadataURL + + logger.Infof(ctx, "Metadata endpoint is %s", result.oauth2MetadataURL) + + result.oidcMetadataURL, err = url.Parse(OIdCMetadataEndpoint) + if err != nil { + logger.Errorf(ctx, "Error parsing oidc metadata URL %s", err) + return Context{}, errors.Wrapf(ErrAuthContext, err, "Error parsing metadata URL") + } + + logger.Infof(ctx, "Metadata endpoint is %s", result.oidcMetadataURL) // Construct the URL object for the user info endpoint if applicable if options.IdpUserInfoEndpoint != "" { @@ -158,14 +177,13 @@ func GetOauth2Config(options config.OAuthOptions) (oauth2.Config, error) { if err != nil { return oauth2.Config{}, err } + secret := strings.TrimSuffix(string(secretBytes), "\n") return oauth2.Config{ RedirectURL: options.CallbackURL, ClientID: options.ClientID, ClientSecret: secret, - // Offline access needs to be specified in order to return a refresh token in the exchange. - // TODO: Second parameter is IDP specific - move to config. Also handle case where a refresh token is not allowed - Scopes: []string{OidcScope, OfflineAccessType, ProfileScope}, + Scopes: options.Scopes, Endpoint: oauth2.Endpoint{ AuthURL: options.AuthorizeURL, TokenURL: options.TokenURL, diff --git a/pkg/auth/config/config.go b/pkg/auth/config/config.go index dde837a01..dcc5d6d39 100644 --- a/pkg/auth/config/config.go +++ b/pkg/auth/config/config.go @@ -58,6 +58,11 @@ type OAuthOptions struct { // into the realm of authorization rather than authentication. DisableForHTTP bool `json:"disableForHttp"` DisableForGrpc bool `json:"disableForGrpc"` + + // Provides a list of scopes to request from the IDP when authenticating. Default value requests claims that should + // be supported by any OIdC server. Refer to https://openid.net/specs/openid-connect-core-1_0.html#ScopeClaims for + // a complete list. Other providers might support additional scopes that you can define in a config. + Scopes []string `json:"scopes"` } type Claims struct { diff --git a/pkg/auth/constants.go b/pkg/auth/constants.go index 707ecb77e..58540510f 100644 --- a/pkg/auth/constants.go +++ b/pkg/auth/constants.go @@ -3,15 +3,14 @@ package auth // OAuth2 Parameters const CsrfFormKey = "state" const AuthorizationResponseCodeType = "code" -const OidcScope = "openid" -const ProfileScope = "profile" const RefreshToken = "refresh_token" const DefaultAuthorizationHeader = "authorization" const BearerScheme = "Bearer" // https://tools.ietf.org/html/rfc8414 // This should be defined without a leading slash. If there is one, the url library's ResolveReference will make it a root path -const MetadataEndpoint = ".well-known/oauth-authorization-server" +const OAuth2MetadataEndpoint = ".well-known/oauth-authorization-server" -// IDP specific -const OfflineAccessType = "offline_access" +// https://openid.net/specs/openid-connect-discovery-1_0.html +// This should be defined without a leading slash. If there is one, the url library's ResolveReference will make it a root path +const OIdCMetadataEndpoint = ".well-known/openid-configuration" diff --git a/pkg/auth/cookie.go b/pkg/auth/cookie.go index 848e20f35..70e7436b3 100644 --- a/pkg/auth/cookie.go +++ b/pkg/auth/cookie.go @@ -17,13 +17,18 @@ import ( const ( // #nosec - accessTokenCookieName = "flyte_jwt" + accessTokenCookieName = "flyte_at" // #nosec - refreshTokenCookieName = "flyte_refresh" + idTokenCookieName = "flyte_idt" + // #nosec + refreshTokenCookieName = "flyte_rt" // #nosec csrfStateCookieName = "flyte_csrf_state" // #nosec redirectURLCookieName = "flyte_redirect_location" + + // #nosec + idTokenExtra = "id_token" ) const ( @@ -54,6 +59,33 @@ func NewSecureCookie(cookieName, value string, hashKey, blockKey []byte) (http.C return http.Cookie{}, errors.Wrapf(ErrSecureCookie, err, "Error creating secure cookie") } +func retrieveSecureCookie(ctx context.Context, request *http.Request, cookieName string, hashKey, blockKey []byte) (string, error) { + cookie, err := request.Cookie(cookieName) + if err != nil { + logger.Infof(ctx, "Could not detect existing cookie [%v]. Error: %v", cookieName, err) + return "", errors.Wrapf(ErrTokenNil, err, "Failure to retrieve cookie [%v]", cookieName) + } + + if cookie == nil { + logger.Infof(ctx, "Retrieved empty cookie [%v].", cookieName) + return "", errors.Errorf(ErrTokenNil, "Retrieved empty cookie [%v]", cookieName) + } + + logger.Debugf(ctx, "Existing [%v] cookie found", cookieName) + token, err := ReadSecureCookie(ctx, *cookie, hashKey, blockKey) + if err != nil { + logger.Errorf(ctx, "Error reading existing secure cookie [%v]. Error: %s", cookieName, err) + return "", errors.Errorf(ErrTokenNil, "Error reading existing secure cookie [%v]. Error: %s", cookieName, err) + } + + if len(token) == 0 { + logger.Errorf(ctx, "Read empty token from secure cookie [%v].", cookieName) + return "", errors.Errorf(ErrTokenNil, "Read empty token from secure cookie [%v].", cookieName) + } + + return token, nil +} + func ReadSecureCookie(ctx context.Context, cookie http.Cookie, hashKey, blockKey []byte) (string, error) { var s = securecookie.New(hashKey, blockKey) var value string diff --git a/pkg/auth/cookie_manager.go b/pkg/auth/cookie_manager.go index 7f228b8a0..f490d6465 100644 --- a/pkg/auth/cookie_manager.go +++ b/pkg/auth/cookie_manager.go @@ -20,6 +20,8 @@ const ( ErrB64Decoding errors.ErrorCode = "BINARY_DECODING_FAILED" // #nosec ErrTokenNil errors.ErrorCode = "EMPTY_OAUTH_TOKEN" + // #nosec + ErrNoIDToken errors.ErrorCode = "NO_ID_TOKEN_IN_RESPONSE" ) func NewCookieManager(ctx context.Context, hashKeyEncoded, blockKeyEncoded string) (CookieManager, error) { @@ -29,6 +31,7 @@ func NewCookieManager(ctx context.Context, hashKeyEncoded, blockKeyEncoded strin if err != nil { return CookieManager{}, errors.Wrapf(ErrB64Decoding, err, "Error decoding hash key bytes") } + blockKey, err := base64.RawStdEncoding.DecodeString(blockKeyEncoded) if err != nil { return CookieManager{}, errors.Wrapf(ErrB64Decoding, err, "Error decoding block key bytes") @@ -41,32 +44,30 @@ func NewCookieManager(ctx context.Context, hashKeyEncoded, blockKeyEncoded strin } // TODO: Separate refresh token from access token, remove named returns, and use stdlib errors. -func (c CookieManager) RetrieveTokenValues(ctx context.Context, request *http.Request) (accessToken string, +// RetrieveTokenValues retrieves id, access and refresh tokens from cookies if they exist. The existence of a refresh token +// in a cookie is optional and hence failure to find or read that cookie is tolerated. An error is returned in case of failure +// to retrieve and read either the id or the access tokens. +func (c CookieManager) RetrieveTokenValues(ctx context.Context, request *http.Request) (idToken, accessToken, refreshToken string, err error) { - jwtCookie, err := request.Cookie(accessTokenCookieName) - if err != nil || jwtCookie == nil { - logger.Errorf(ctx, "Could not detect existing access token cookie") - return - } - logger.Debugf(ctx, "Existing JWT cookie found") - accessToken, err = ReadSecureCookie(ctx, *jwtCookie, c.hashKey, c.blockKey) + idToken, err = retrieveSecureCookie(ctx, request, idTokenCookieName, c.hashKey, c.blockKey) if err != nil { - logger.Errorf(ctx, "Error reading existing secure JWT cookie %s", err) - return + return "", "", "", err } - refreshCookie, err := request.Cookie(refreshTokenCookieName) - if err != nil || refreshCookie == nil { - logger.Debugf(ctx, "Could not detect existing access token cookie") - return + accessToken, err = retrieveSecureCookie(ctx, request, accessTokenCookieName, c.hashKey, c.blockKey) + if err != nil { + return "", "", "", err } - logger.Debugf(ctx, "Existing refresh cookie found") - refreshToken, err = ReadSecureCookie(ctx, *refreshCookie, c.hashKey, c.blockKey) + + refreshToken, err = retrieveSecureCookie(ctx, request, refreshTokenCookieName, c.hashKey, c.blockKey) if err != nil { - logger.Errorf(ctx, "Error reading existing secure refresh cookie %s", err) - return + // Refresh tokens are optional. Depending on the auth url (IdP specific) we might or might not receive a refresh + // token. In case we do not, we will just have to redirect to IdP whenever access/id tokens expire. + logger.Infof(ctx, "Refresh token doesn't exist or failed to read it. Ignoring this error. Error: %v", err) + err = nil } + return } @@ -76,22 +77,37 @@ func (c CookieManager) SetTokenCookies(ctx context.Context, writer http.Response return errors.Errorf(ErrTokenNil, "Attempting to set cookies with nil token") } - jwtCookie, err := NewSecureCookie(accessTokenCookieName, token.AccessToken, c.hashKey, c.blockKey) + atCookie, err := NewSecureCookie(accessTokenCookieName, token.AccessToken, c.hashKey, c.blockKey) if err != nil { - logger.Errorf(ctx, "Error generating encrypted JWT cookie %s", err) + logger.Errorf(ctx, "Error generating encrypted accesstoken cookie %s", err) return err } - http.SetCookie(writer, &jwtCookie) + + http.SetCookie(writer, &atCookie) + + if idTokenRaw, converted := token.Extra(idTokenExtra).(string); converted { + idCookie, err := NewSecureCookie(idTokenCookieName, idTokenRaw, c.hashKey, c.blockKey) + if err != nil { + logger.Errorf(ctx, "Error generating encrypted id token cookie %s", err) + return err + } + + http.SetCookie(writer, &idCookie) + } else { + logger.Errorf(ctx, "Response does not contain an id_token.") + return errors.Errorf(ErrNoIDToken, "Response does not contain an id_token.") + } // Set the refresh cookie if there is a refresh token if token.RefreshToken != "" { refreshCookie, err := NewSecureCookie(refreshTokenCookieName, token.RefreshToken, c.hashKey, c.blockKey) if err != nil { - logger.Errorf(ctx, "Error generating encrypted refresh cookie %s", err) + logger.Errorf(ctx, "Error generating encrypted refresh token cookie %s", err) return err } http.SetCookie(writer, &refreshCookie) } + return nil } diff --git a/pkg/auth/cookie_manager_test.go b/pkg/auth/cookie_manager_test.go index 44f052ee9..28b377e07 100644 --- a/pkg/auth/cookie_manager_test.go +++ b/pkg/auth/cookie_manager_test.go @@ -21,18 +21,23 @@ func TestCookieManager_SetTokenCookies(t *testing.T) { manager, err := NewCookieManager(ctx, hashKeyEncoded, blockKeyEncoded) assert.NoError(t, err) - token := oauth2.Token{ + token := &oauth2.Token{ AccessToken: "access", RefreshToken: "refresh", } + token = token.WithExtra(map[string]interface{}{ + "id_token": "id token", + }) + w := httptest.NewRecorder() - err = manager.SetTokenCookies(ctx, w, &token) + err = manager.SetTokenCookies(ctx, w, token) assert.NoError(t, err) fmt.Println(w.Header().Get("Set-Cookie")) c := w.Result().Cookies() - assert.Equal(t, "flyte_jwt", c[0].Name) - assert.Equal(t, "flyte_refresh", c[1].Name) + assert.Equal(t, "flyte_at", c[0].Name) + assert.Equal(t, "flyte_idt", c[1].Name) + assert.Equal(t, "flyte_rt", c[2].Name) } func TestCookieManager_RetrieveTokenValues(t *testing.T) { @@ -44,23 +49,29 @@ func TestCookieManager_RetrieveTokenValues(t *testing.T) { manager, err := NewCookieManager(ctx, hashKeyEncoded, blockKeyEncoded) assert.NoError(t, err) - token := oauth2.Token{ + token := &oauth2.Token{ AccessToken: "access", RefreshToken: "refresh", } + token = token.WithExtra(map[string]interface{}{ + "id_token": "id token", + }) + w := httptest.NewRecorder() - err = manager.SetTokenCookies(ctx, w, &token) + err = manager.SetTokenCookies(ctx, w, token) assert.NoError(t, err) cookies := w.Result().Cookies() req, err := http.NewRequest("GET", "/api/v1/projects", nil) assert.NoError(t, err) - req.AddCookie(cookies[0]) - req.AddCookie(cookies[1]) + for _, c := range cookies { + req.AddCookie(c) + } - access, refresh, err := manager.RetrieveTokenValues(ctx, req) + idToken, access, refresh, err := manager.RetrieveTokenValues(ctx, req) assert.NoError(t, err) + assert.Equal(t, "id token", idToken) assert.Equal(t, "access", access) assert.Equal(t, "refresh", refresh) } diff --git a/pkg/auth/handlers.go b/pkg/auth/handlers.go index 3ea1092ff..c9a83386d 100644 --- a/pkg/auth/handlers.go +++ b/pkg/auth/handlers.go @@ -17,7 +17,6 @@ import ( "github.com/lyft/flytestdlib/contextutils" "github.com/lyft/flytestdlib/errors" "github.com/lyft/flytestdlib/logger" - "golang.org/x/oauth2" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/metadata" @@ -40,28 +39,26 @@ func RefreshTokensIfExists(ctx context.Context, authContext interfaces.Authentic return func(writer http.ResponseWriter, request *http.Request) { // Since we only do one thing if there are no errors anywhere along the chain, we can save code by just // using one variable and checking for errors at the end. - var err error - accessToken, refreshToken, err := authContext.CookieManager().RetrieveTokenValues(ctx, request) - - if err == nil && accessToken != "" && refreshToken != "" { - _, e := ParseAndValidate(ctx, authContext.Claims(), accessToken, authContext.OidcProvider()) - err = e - if err != nil && errors.IsCausedBy(err, ErrTokenExpired) { - logger.Debugf(ctx, "Expired access token found, attempting to refresh") + idToken, accessToken, refreshToken, err := authContext.CookieManager().RetrieveTokenValues(ctx, request) + if err == nil { + _, err = ParseAndValidate(ctx, authContext.Claims(), idToken, authContext.OidcProvider()) + if err != nil && errors.IsCausedBy(err, ErrTokenExpired) && len(refreshToken) > 0 { + logger.Debugf(ctx, "Expired id token found, attempting to refresh") newToken, e := GetRefreshedToken(ctx, authContext.OAuth2Config(), accessToken, refreshToken) err = e if err == nil { - logger.Debugf(ctx, "Access token refreshed. Saving new tokens into cookies.") + logger.Debugf(ctx, "Tokens are refreshed. Saving new tokens into cookies.") err = authContext.CookieManager().SetTokenCookies(ctx, writer, newToken) } } } if err != nil { - logger.Errorf(ctx, "Non-expiration error in refresh token handler %s, redirecting to login handler", err) + logger.Errorf(ctx, "Non-expiration error in refresh token handler, redirecting to login handler. Error: %s", err) handlerFunc(writer, request) return } + redirectURL := getAuthFlowEndRedirect(ctx, authContext, request) http.Redirect(writer, request, redirectURL, http.StatusTemporaryRedirect) } @@ -101,11 +98,7 @@ func GetCallbackHandler(ctx context.Context, authContext interfaces.Authenticati return } - // TODO: The second parameter is IDP specific but seems to be convention, make configurable anyways. - // The second parameter is necessary to get the initial refresh token - offlineAccessParam := oauth2.SetAuthURLParam(RefreshToken, OfflineAccessType) - - token, err := authContext.OAuth2Config().Exchange(ctx, authorizationCode, offlineAccessParam) + token, err := authContext.OAuth2Config().Exchange(ctx, authorizationCode) if err != nil { logger.Errorf(ctx, "Error when exchanging code %s", err) writer.WriteHeader(http.StatusForbidden) @@ -180,6 +173,7 @@ func GetAuthenticationInterceptor(authContext interfaces.AuthenticationContext) return ctx, status.Errorf(codes.Unauthenticated, "no email or empty email found") } } + if token != nil { newCtx := WithUserEmail(context.WithValue(ctx, bearerTokenContextKey, token), token.Subject) newCtx = WithAuditFields(newCtx, token.Audience, token.IssuedAt) @@ -214,9 +208,8 @@ func WithAuditFields(ctx context.Context, clientIds []string, tokenIssuedAt time func GetHTTPRequestCookieToMetadataHandler(authContext interfaces.AuthenticationContext) HTTPRequestToMetadataAnnotator { return func(ctx context.Context, request *http.Request) metadata.MD { // TODO: Improve error handling - accessToken, _, _ := authContext.CookieManager().RetrieveTokenValues(ctx, request) - if accessToken == "" { - + idToken, _, _, _ := authContext.CookieManager().RetrieveTokenValues(ctx, request) + if len(idToken) == 0 { // If no token was found in the cookies, look for an authorization header, starting with a potentially // custom header set in the Config object if authContext.Options().HTTPAuthorizationHeader != "" { @@ -233,7 +226,7 @@ func GetHTTPRequestCookieToMetadataHandler(authContext interfaces.Authentication return nil } return metadata.MD{ - DefaultAuthorizationHeader: []string{fmt.Sprintf("%s %s", BearerScheme, accessToken)}, + DefaultAuthorizationHeader: []string{fmt.Sprintf("%s %s", BearerScheme, idToken)}, } } } @@ -254,14 +247,14 @@ func GetHTTPMetadataTaggingHandler(authContext interfaces.AuthenticationContext) func GetMeEndpointHandler(ctx context.Context, authCtx interfaces.AuthenticationContext) http.HandlerFunc { idpUserInfoEndpoint := authCtx.GetUserInfoURL().String() return func(writer http.ResponseWriter, request *http.Request) { - access, _, err := authCtx.CookieManager().RetrieveTokenValues(ctx, request) + _, accessToken, _, err := authCtx.CookieManager().RetrieveTokenValues(ctx, request) if err != nil { http.Error(writer, "Error decoding identify token, try /login in again", http.StatusUnauthorized) return } // TODO: Investigate improving transparency of errors. The errors from this call may be just a local error, or may // be an error from the HTTP request to the IDP. In the latter case, consider passing along the error code/msg. - userInfo, err := postToIdp(ctx, authCtx.GetHTTPClient(), idpUserInfoEndpoint, access) + userInfo, err := postToIdp(ctx, authCtx.GetHTTPClient(), idpUserInfoEndpoint, accessToken) if err != nil { logger.Errorf(ctx, "Error getting user info from IDP %s", err) http.Error(writer, "Error getting user info from IDP", http.StatusFailedDependency) @@ -284,9 +277,18 @@ func GetMeEndpointHandler(ctx context.Context, authCtx interfaces.Authentication // This returns a handler that will redirect (303) to the well-known metadata endpoint for the OAuth2 authorization server // See https://tools.ietf.org/html/rfc8414 for more information. -func GetMetadataEndpointRedirectHandler(ctx context.Context, authCtx interfaces.AuthenticationContext) http.HandlerFunc { +func GetOAuth2MetadataEndpointRedirectHandler(ctx context.Context, authCtx interfaces.AuthenticationContext) http.HandlerFunc { + return func(writer http.ResponseWriter, request *http.Request) { + metadataURL := authCtx.GetBaseURL().ResolveReference(authCtx.GetOAuth2MetadataURL()) + http.Redirect(writer, request, metadataURL.String(), http.StatusSeeOther) + } +} + +// This returns a handler that will redirect (303) to the well-known metadata endpoint for the OAuth2 authorization server +// See https://tools.ietf.org/html/rfc8414 for more information. +func GetOIdCMetadataEndpointRedirectHandler(ctx context.Context, authCtx interfaces.AuthenticationContext) http.HandlerFunc { return func(writer http.ResponseWriter, request *http.Request) { - metadataURL := authCtx.GetBaseURL().ResolveReference(authCtx.GetMetadataURL()) + metadataURL := authCtx.GetBaseURL().ResolveReference(authCtx.GetOIdCMetadataURL()) http.Redirect(writer, request, metadataURL.String(), http.StatusSeeOther) } } diff --git a/pkg/auth/handlers_test.go b/pkg/auth/handlers_test.go index 8681f8a12..4f551ed60 100644 --- a/pkg/auth/handlers_test.go +++ b/pkg/auth/handlers_test.go @@ -49,12 +49,19 @@ func TestGetHTTPRequestCookieToMetadataHandler(t *testing.T) { assert.NoError(t, err) mockAuthCtx := mocks.AuthenticationContext{} mockAuthCtx.On("CookieManager").Return(&cookieManager) + mockAuthCtx.OnOptions().Return(config.OAuthOptions{}) handler := GetHTTPRequestCookieToMetadataHandler(&mockAuthCtx) req, err := http.NewRequest("GET", "/api/v1/projects", nil) assert.NoError(t, err) - jwtCookie, err := NewSecureCookie(accessTokenCookieName, "a.b.c", cookieManager.hashKey, cookieManager.blockKey) + + accessTokenCookie, err := NewSecureCookie(accessTokenCookieName, "a.b.c", cookieManager.hashKey, cookieManager.blockKey) + assert.NoError(t, err) + req.AddCookie(&accessTokenCookie) + + idCookie, err := NewSecureCookie(idTokenCookieName, "a.b.c", cookieManager.hashKey, cookieManager.blockKey) assert.NoError(t, err) - req.AddCookie(&jwtCookie) + req.AddCookie(&idCookie) + assert.Equal(t, "Bearer a.b.c", handler(ctx, req)["authorization"][0]) } @@ -92,12 +99,12 @@ func TestGetMetadataEndpointRedirectHandler(t *testing.T) { ctx := context.Background() baseURL, err := url.Parse("http://www.google.com") assert.NoError(t, err) - metadataPath, err := url.Parse(MetadataEndpoint) + metadataPath, err := url.Parse(OAuth2MetadataEndpoint) assert.NoError(t, err) mockAuthCtx := mocks.AuthenticationContext{} - mockAuthCtx.On("GetBaseURL").Return(baseURL) - mockAuthCtx.On("GetMetadataURL").Return(metadataPath) - handler := GetMetadataEndpointRedirectHandler(ctx, &mockAuthCtx) + mockAuthCtx.OnGetBaseURL().Return(baseURL) + mockAuthCtx.OnGetOAuth2MetadataURL().Return(metadataPath) + handler := GetOAuth2MetadataEndpointRedirectHandler(ctx, &mockAuthCtx) req, err := http.NewRequest("GET", "/xyz", nil) assert.NoError(t, err) w := httptest.NewRecorder() diff --git a/pkg/auth/interfaces/context.go b/pkg/auth/interfaces/context.go index 485357349..6d5452fc1 100644 --- a/pkg/auth/interfaces/context.go +++ b/pkg/auth/interfaces/context.go @@ -21,6 +21,7 @@ type AuthenticationContext interface { Options() config.OAuthOptions GetUserInfoURL() *url.URL GetBaseURL() *url.URL - GetMetadataURL() *url.URL + GetOAuth2MetadataURL() *url.URL + GetOIdCMetadataURL() *url.URL GetHTTPClient() *http.Client } diff --git a/pkg/auth/interfaces/cookie.go b/pkg/auth/interfaces/cookie.go index a76dc36ee..27232094c 100644 --- a/pkg/auth/interfaces/cookie.go +++ b/pkg/auth/interfaces/cookie.go @@ -8,7 +8,7 @@ import ( ) type CookieHandler interface { - RetrieveTokenValues(ctx context.Context, request *http.Request) (accessToken string, refreshToken string, err error) + RetrieveTokenValues(ctx context.Context, request *http.Request) (idToken, accessToken, refreshToken string, err error) SetTokenCookies(ctx context.Context, writer http.ResponseWriter, token *oauth2.Token) error DeleteCookies(ctx context.Context, writer http.ResponseWriter) } diff --git a/pkg/auth/interfaces/mocks/authentication_context.go b/pkg/auth/interfaces/mocks/authentication_context.go index 34b6ef044..32870207e 100644 --- a/pkg/auth/interfaces/mocks/authentication_context.go +++ b/pkg/auth/interfaces/mocks/authentication_context.go @@ -157,26 +157,60 @@ func (_m *AuthenticationContext) GetHTTPClient() *http.Client { return r0 } -type AuthenticationContext_GetMetadataURL struct { +type AuthenticationContext_GetOAuth2MetadataURL struct { *mock.Call } -func (_m AuthenticationContext_GetMetadataURL) Return(_a0 *url.URL) *AuthenticationContext_GetMetadataURL { - return &AuthenticationContext_GetMetadataURL{Call: _m.Call.Return(_a0)} +func (_m AuthenticationContext_GetOAuth2MetadataURL) Return(_a0 *url.URL) *AuthenticationContext_GetOAuth2MetadataURL { + return &AuthenticationContext_GetOAuth2MetadataURL{Call: _m.Call.Return(_a0)} } -func (_m *AuthenticationContext) OnGetMetadataURL() *AuthenticationContext_GetMetadataURL { - c := _m.On("GetMetadataURL") - return &AuthenticationContext_GetMetadataURL{Call: c} +func (_m *AuthenticationContext) OnGetOAuth2MetadataURL() *AuthenticationContext_GetOAuth2MetadataURL { + c := _m.On("GetOAuth2MetadataURL") + return &AuthenticationContext_GetOAuth2MetadataURL{Call: c} } -func (_m *AuthenticationContext) OnGetMetadataURLMatch(matchers ...interface{}) *AuthenticationContext_GetMetadataURL { - c := _m.On("GetMetadataURL", matchers...) - return &AuthenticationContext_GetMetadataURL{Call: c} +func (_m *AuthenticationContext) OnGetOAuth2MetadataURLMatch(matchers ...interface{}) *AuthenticationContext_GetOAuth2MetadataURL { + c := _m.On("GetOAuth2MetadataURL", matchers...) + return &AuthenticationContext_GetOAuth2MetadataURL{Call: c} } -// GetMetadataURL provides a mock function with given fields: -func (_m *AuthenticationContext) GetMetadataURL() *url.URL { +// GetOAuth2MetadataURL provides a mock function with given fields: +func (_m *AuthenticationContext) GetOAuth2MetadataURL() *url.URL { + ret := _m.Called() + + var r0 *url.URL + if rf, ok := ret.Get(0).(func() *url.URL); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*url.URL) + } + } + + return r0 +} + +type AuthenticationContext_GetOIdCMetadataURL struct { + *mock.Call +} + +func (_m AuthenticationContext_GetOIdCMetadataURL) Return(_a0 *url.URL) *AuthenticationContext_GetOIdCMetadataURL { + return &AuthenticationContext_GetOIdCMetadataURL{Call: _m.Call.Return(_a0)} +} + +func (_m *AuthenticationContext) OnGetOIdCMetadataURL() *AuthenticationContext_GetOIdCMetadataURL { + c := _m.On("GetOIdCMetadataURL") + return &AuthenticationContext_GetOIdCMetadataURL{Call: c} +} + +func (_m *AuthenticationContext) OnGetOIdCMetadataURLMatch(matchers ...interface{}) *AuthenticationContext_GetOIdCMetadataURL { + c := _m.On("GetOIdCMetadataURL", matchers...) + return &AuthenticationContext_GetOIdCMetadataURL{Call: c} +} + +// GetOIdCMetadataURL provides a mock function with given fields: +func (_m *AuthenticationContext) GetOIdCMetadataURL() *url.URL { ret := _m.Called() var r0 *url.URL diff --git a/pkg/auth/token.go b/pkg/auth/token.go index 6eb7a4189..642222c87 100644 --- a/pkg/auth/token.go +++ b/pkg/auth/token.go @@ -38,12 +38,14 @@ func GetRefreshedToken(ctx context.Context, oauth *oauth2.Config, accessToken, r return newToken, nil } -func ParseAndValidate(ctx context.Context, claims config.Claims, accessToken string, +func ParseAndValidate(ctx context.Context, claims config.Claims, rawIDToken string, provider *oidc.Provider) (*oidc.IDToken, error) { - var verifier = provider.Verifier(&oidc.Config{ClientID: claims.Audience}) + var verifier = provider.Verifier(&oidc.Config{ + ClientID: claims.Audience, + }) - idToken, err := verifier.Verify(ctx, accessToken) + idToken, err := verifier.Verify(ctx, rawIDToken) if err != nil { logger.Debugf(ctx, "JWT parsing with claims failed %s", err) flyteErr := errors.Wrapf(ErrJwtValidation, err, "jwt parse with claims failed") @@ -51,8 +53,10 @@ func ParseAndValidate(ctx context.Context, claims config.Claims, accessToken str if strings.Contains(err.Error(), "token is expired") { return idToken, errors.Wrapf(ErrTokenExpired, flyteErr, "token is expired") } + return idToken, flyteErr } + return idToken, nil } @@ -66,6 +70,7 @@ func GetAndValidateTokenObjectFromContext(ctx context.Context, claims config.Cla logger.Debugf(ctx, "Could not retrieve bearer token from metadata %v", err) return nil, errors.Wrapf(ErrJwtValidation, err, "Could not retrieve bearer token from metadata") } + if tokenStr == "" { logger.Debugf(ctx, "Found Bearer scheme but token was blank") return nil, errors.Errorf(ErrJwtValidation, "Bearer token is blank") diff --git a/pkg/config/config.go b/pkg/config/config.go index 8809c80ee..512be6446 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -52,6 +52,12 @@ var defaultServerConfig = &ServerConfig{ // Please see the comments in this struct's definition for more information HTTPAuthorizationHeader: "flyte-authorization", GrpcAuthorizationHeader: "flyte-authorization", + // Default claims that should be supported by any OIdC server. Refer to https://openid.net/specs/openid-connect-core-1_0.html#ScopeClaims + // for a complete list. + Scopes: []string{ + "openid", + "profile", + }, }, }, } diff --git a/pkg/rpc/config/flyte_client.go b/pkg/rpc/config/flyte_client.go index eaf78731c..f28a02ea5 100644 --- a/pkg/rpc/config/flyte_client.go +++ b/pkg/rpc/config/flyte_client.go @@ -12,22 +12,26 @@ import ( const ( clientID = "client_id" redirectURI = "redirect_uri" + scopes = "scopes" authMetadataKey = "authorization_metadata_key" ) func HandleFlyteCliConfigFunc(ctx context.Context, cfg *config.ServerConfig) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - configValues := map[string]string{ + configValues := map[string]interface{}{ clientID: cfg.ThirdPartyConfig.FlyteClientConfig.ClientID, redirectURI: cfg.ThirdPartyConfig.FlyteClientConfig.RedirectURI, + scopes: cfg.Security.Oauth.Scopes, authMetadataKey: cfg.Security.Oauth.GrpcAuthorizationHeader, } + configJSON, err := json.Marshal(configValues) if err != nil { logger.Infof(ctx, "Failed to marshal flyte_client config to JSON with err: %v", err) w.WriteHeader(http.StatusInternalServerError) return } + _, err = w.Write(configJSON) if err != nil { logger.Warningf(ctx, "Failed to write config json [%+v] with err: %v", string(configJSON), err) diff --git a/pkg/rpc/config/flyte_client_test.go b/pkg/rpc/config/flyte_client_test.go index 47b2bd333..2c6bbe529 100644 --- a/pkg/rpc/config/flyte_client_test.go +++ b/pkg/rpc/config/flyte_client_test.go @@ -44,5 +44,6 @@ func TestHandleFlyteCliConfigFunc(t *testing.T) { clientID: testClientID, redirectURI: testRedirectURI, authMetadataKey: testAuthMetadataKey, + scopes: "", }, responseBodyMap) } From 0b778a978762256451b03596420e776f47bf8d00 Mon Sep 17 00:00:00 2001 From: Anand Swaminathan Date: Tue, 2 Mar 2021 15:36:03 -0800 Subject: [PATCH 22/22] Add project and domain to cluster resource templating (#153) * Add project and domain to cluster resource templating Signed-off-by: Anand Swaminathan --- pkg/clusterresource/controller.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/pkg/clusterresource/controller.go b/pkg/clusterresource/controller.go index 89ff4657c..a485499f4 100644 --- a/pkg/clusterresource/controller.go +++ b/pkg/clusterresource/controller.go @@ -11,6 +11,8 @@ import ( "strings" "time" + "github.com/lyft/flyteadmin/pkg/repositories/models" + v1 "k8s.io/api/rbac/v1" "github.com/lyft/flyteadmin/pkg/manager/impl/resources" @@ -40,6 +42,8 @@ import ( ) const namespaceVariable = "namespace" +const projectVariable = "project" +const domainVariable = "domain" const templateVariableFormat = "{{ %s }}" const replaceAllInstancesOfString = -1 @@ -121,7 +125,7 @@ func populateTemplateValues(data map[string]runtimeInterfaces.DataSource) (templ templateValues := make(templateValuesType, len(data)) collectedErrs := make([]error, 0) for templateVar, dataSource := range data { - if templateVar == namespaceVariable { + if templateVar == namespaceVariable || templateVar == projectVariable || templateVar == domainVariable { // The namespace variable is specifically reserved for system use only. collectedErrs = append(collectedErrs, errors.NewFlyteAdminErrorf(codes.InvalidArgument, "Cannot assign namespace template value in user data")) @@ -220,7 +224,7 @@ func (c *controller) getCustomTemplateValues( // 2) substitute templatized variables with their resolved values // 3) decode the output of the above into a kubernetes resource // 4) create the resource on the kubernetes cluster and cache successful outcomes -func (c *controller) syncNamespace(ctx context.Context, namespace NamespaceName, +func (c *controller) syncNamespace(ctx context.Context, project models.Project, domain runtimeInterfaces.Domain, namespace NamespaceName, templateValues, customTemplateValues templateValuesType) error { templateDir := c.config.ClusterResourceConfiguration().GetTemplatePath() if c.lastAppliedTemplateDir != templateDir { @@ -270,6 +274,9 @@ func (c *controller) syncNamespace(ctx context.Context, namespace NamespaceName, // First, add the special case namespace template which is always substituted by the system // rather than fetched via a user-specified source. templateValues[fmt.Sprintf(templateVariableFormat, namespaceVariable)] = namespace + templateValues[fmt.Sprintf(templateVariableFormat, projectVariable)] = project.Name + templateValues[fmt.Sprintf(templateVariableFormat, domainVariable)] = domain.Name + var config = string(template) for templateKey, templateValue := range templateValues { config = strings.Replace(config, templateKey, templateValue, replaceAllInstancesOfString) @@ -391,7 +398,7 @@ func (c *controller) Sync(ctx context.Context) error { logger.Warningf(ctx, "Failed to get custom template values for %s with err: %v", namespace, err) errs = append(errs, err) } - err = c.syncNamespace(ctx, namespace, templateValues, customTemplateValues) + err = c.syncNamespace(ctx, project, domain, namespace, templateValues, customTemplateValues) if err != nil { logger.Warningf(ctx, "Failed to create cluster resources for namespace [%s] with err: %v", namespace, err) c.metrics.ResourceAddErrors.Inc()