[DDO-3719] Role Propagation #minor

go-shared/pkg/utils/substitute_suffix.go

@@ -0,0 +1,13 @@
+package utils
+
+import "strings"
+
+func SubstituteSuffix(s string, suffixesToReplace []string, replacement string) string {
+	for _, suffix := range suffixesToReplace {
+		if strings.HasSuffix(s, suffix) {
+			s = strings.TrimSuffix(s, suffix) + replacement
+			break
+		}
+	}
+	return s
+}

go-shared/pkg/utils/substitute_suffix_test.go

@@ -0,0 +1,59 @@
+package utils
+
+import "testing"
+
+func TestSubstituteSuffix(t *testing.T) {
+	type args struct {
+		s                 string
+		replacement       string
+		suffixesToReplace []string
+	}
+	tests := []struct {
+		name string
+		args args
+		want string
+	}{
+		{
+			name: "no old domains",
+			args: args{
+				s:           "[email protected]",
+				replacement: "example.org",
+			},
+			want: "[email protected]",
+		},
+		{
+			name: "single old domain",
+			args: args{
+				s:                 "[email protected]",
+				suffixesToReplace: []string{"example.com"},
+				replacement:       "example.org",
+			},
+			want: "[email protected]",
+		},
+		{
+			name: "multiple old domains",
+			args: args{
+				s:                 "[email protected]",
+				suffixesToReplace: []string{"example.com", "example.net"},
+				replacement:       "example.org",
+			},
+			want: "[email protected]",
+		},
+		{
+			name: "no match",
+			args: args{
+				s:                 "[email protected]",
+				suffixesToReplace: []string{"example.net"},
+				replacement:       "example.org",
+			},
+			want: "[email protected]",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := SubstituteSuffix(tt.args.s, tt.args.suffixesToReplace, tt.args.replacement); got != tt.want {
+				t.Errorf("SubstituteSuffix() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

sherlock/.mockery.yaml

@@ -54,3 +54,13 @@ packages:
   github.com/broadinstitute/sherlock/sherlock/internal/middleware/authentication/gha_oidc:
     interfaces:
       mockableVerifier:
+  github.com/broadinstitute/sherlock/sherlock/internal/role_propagation:
+    interfaces:
+      propagator:
+  github.com/broadinstitute/sherlock/sherlock/internal/role_propagation/intermediary_user:
+    interfaces:
+      Identifier:
+      Fields:
+  github.com/broadinstitute/sherlock/sherlock/internal/role_propagation/propagation_engines:
+    interfaces:
+      PropagationEngine:

sherlock/config/default_config.yaml

@@ -245,3 +245,84 @@ beehive:
 self:
     overrideEmail:
     overrideSubjectID:
+
+rolePropagation:
+  # If true, Sherlock's boot process will include configuring the standard array of role propagators
+  # (configured in the "propagators" section below). If false, the array of propagators will typically
+  # be empty, so propagation will have no effect.
+  enable: false
+  # If true, roles will be propagated asynchronously after requests to the role and role assignment
+  # endpoints. If false, roles will be propagated synchronously before those requests are completed.
+  asynchronous: true
+  # The duration that an individual propagator should be able to run for a single role before being
+  # forcibly shut down with an error. This can be overridden for individual propagators by specifying
+  # a "timeout" field in their configuration.
+  defaultTimeout: 5m
+  # The duration after which Sherlock will consider a role's propagation to be stale and in need of
+  # re-propagation. This measures against the end of the last propagation (regardless of success).
+  driftAlignmentStaleThreshold: 5m
+  propagators:
+
+    devFirecloudGroup:
+      enable: false
+      # The domain of the Google Workspace, assumed to be the email domain of all members. This should
+      # not contain a leading "@".
+      workspaceDomain: "test.firecloud.org"
+      # Suffixes of Sherlock users' emails that should be swapped out with "@"+workspaceDomain to match
+      # Sherlock users to Google Workspace users. This must contain a "@".
+      userEmailSuffixesToReplace:
+        - "@broadinstitute.org"
+
+    qaFirecloudGroup:
+      enable: false
+      # The domain of the Google Workspace, assumed to be the email domain of all members. This should
+      # not contain a leading "@".
+      workspaceDomain: "qa.firecloud.org"
+      # Suffixes of Sherlock users' emails that should be swapped out with "@"+workspaceDomain to match
+      # Sherlock users to Google Workspace users. This must contain a "@".
+      userEmailSuffixesToReplace:
+        - "@broadinstitute.org"
+
+    prodFirecloudGroup:
+      enable: false
+      # The domain of the Google Workspace, assumed to be the email domain of all members. This should
+      # not contain a leading "@".
+      workspaceDomain: "firecloud.org"
+      # Suffixes of Sherlock users' emails that should be swapped out with "@"+workspaceDomain to match
+      # Sherlock users to Google Workspace users. This must contain a "@".
+      userEmailSuffixesToReplace:
+        - "@broadinstitute.org"
+
+    devAzureGroup:
+      enable: false
+      # The client ID of the Azure AD app to use for authentication.
+      clientID:
+      # The UUID of the Azure AD tenant to work with.
+      tenantID: fad90753-2022-4456-9b0a-c7e5b934e408 # azure.dev.envs-terra.bio
+      # The path on disk that Sherlock should expect to find a token for federated workload identity.
+      tokenFilePath: /azure-federation/projected-ksa-token.jwt
+      # The suffix of all member emails. This can be thought of as a filter for what Azure users Sherlock
+      # will attempt to propagate roles to. This may contain a "@" (especially useful for "#EXT#@" emails);
+      # if it does, then the userEmailSuffixesToReplace must as well.
+      memberEmailSuffix: "_broadinstitute.org#EXT#@devazureterra.onmicrosoft.com"
+      # Suffixes of Sherlock users' emails that should be swapped out with the memberEmailSuffix to match
+      # Sherlock users to Azure Entra ID users.
+      userEmailSuffixesToReplace:
+        - "@broadinstitute.org"
+
+    prodAzureGroup:
+      enable: false
+      # The client ID of the Azure AD app to use for authentication.
+      clientID:
+      # The UUID of the Azure AD tenant to work with.
+      tenantID: 66bb90ac-8857-4a8a-aa0a-be2186dfa5f9 # firecloud.org
+      # The path on disk that Sherlock should expect to find a token for federated workload identity.
+      tokenFilePath: /azure-federation/projected-ksa-token.jwt
+      # The suffix of all member emails. This can be thought of as a filter for what Azure users Sherlock
+      # will attempt to propagate roles to. This may contain a "@" (especially useful for "#EXT#@" emails);
+      # if it does, then the userEmailSuffixesToReplace must as well.
+      memberEmailSuffix: "_broadinstitute.org#EXT#@terraazureprod.onmicrosoft.com"
+      # Suffixes of Sherlock users' emails that should be swapped out with the memberEmailSuffix to match
+      # Sherlock users to Azure Entra ID users.
+      userEmailSuffixesToReplace:
+        - "@broadinstitute.org"

sherlock/config/test_config.yaml

@@ -53,3 +53,23 @@ model:
 self:
   overrideEmail: [email protected]
   overrideSubjectID: sherlock-test
+
+rolePropagation:
+  asynchronous: false
+  propagators:
+    devFirecloudGroupTestDisabled:
+      enable: false
+      workspaceDomain: test.firecloud.org
+
+    devFirecloudGroupTestDefault:
+      enable: true
+      workspaceDomain: test.firecloud.org
+
+    devFirecloudGroupTestConfig:
+      enable: true
+      workspaceDomain: test.firecloud.org
+      timeout: 10s
+      userEmailDomainsToReplace:
+        - broadinstitute.org
+      toleratedUsers:
+        - email: [email protected]

sherlock/db/migrations/000089_role_propagated_at.down.sql

@@ -0,0 +1,2 @@
+alter table roles
+    drop column if exists propagated_at;

sherlock/db/migrations/000089_role_propagated_at.up.sql

@@ -0,0 +1,2 @@
+alter table roles
+    add column if not exists propagated_at timestamp with time zone;

sherlock/db/migrations/000090_role_qa_prod_group_fields.down.sql

@@ -0,0 +1,32 @@
+drop index if exists roles_grants_qa_firecloud_group_unique;
+
+alter table roles
+    drop column if exists grants_qa_firecloud_group;
+
+alter table role_operations
+    drop column if exists from_grants_qa_firecloud_group;
+
+alter table role_operations
+    drop column if exists to_grants_qa_firecloud_group;
+
+drop index if exists roles_grants_prod_firecloud_group_unique;
+
+alter table roles
+    drop column if exists grants_prod_firecloud_group;
+
+alter table role_operations
+    drop column if exists from_grants_prod_firecloud_group;
+
+alter table role_operations
+    drop column if exists to_grants_prod_firecloud_group;
+
+drop index if exists roles_grants_prod_azure_group_unique;
+
+alter table roles
+    drop column if exists grants_prod_azure_group;
+
+alter table role_operations
+    drop column if exists from_grants_prod_azure_group;
+
+alter table role_operations
+    drop column if exists to_grants_prod_azure_group;

sherlock/db/migrations/000090_role_qa_prod_group_fields.up.sql

@@ -0,0 +1,38 @@
+alter table roles
+    add column if not exists grants_qa_firecloud_group text;
+
+create unique index if not exists roles_grants_qa_firecloud_group_unique
+    on roles (grants_qa_firecloud_group)
+    where deleted_at is null and grants_qa_firecloud_group is not null and grants_qa_firecloud_group != '';
+
+alter table role_operations
+    add column if not exists from_grants_qa_firecloud_group text;
+
+alter table role_operations
+    add column if not exists to_grants_qa_firecloud_group text;
+
+alter table roles
+    add column if not exists grants_prod_firecloud_group text;
+
+create unique index if not exists roles_grants_prod_firecloud_group_unique
+    on roles (grants_prod_firecloud_group)
+    where deleted_at is null and grants_prod_firecloud_group is not null and grants_prod_firecloud_group != '';
+
+alter table role_operations
+    add column if not exists from_grants_prod_firecloud_group text;
+
+alter table role_operations
+    add column if not exists to_grants_prod_firecloud_group text;
+
+alter table roles
+    add column if not exists grants_prod_azure_group text;
+
+create unique index if not exists roles_grants_prod_azure_group_unique
+    on roles (grants_prod_azure_group)
+    where deleted_at is null and grants_prod_azure_group is not null and grants_prod_azure_group != '';
+
+alter table role_operations
+    add column if not exists from_grants_prod_azure_group text;
+
+alter table role_operations
+    add column if not exists to_grants_prod_azure_group text;

sherlock/go.mod

@@ -5,6 +5,7 @@ go 1.21
 require (
 	cloud.google.com/go/cloudsqlconn v1.10.1
 	contrib.go.opencensus.io/exporter/prometheus v0.4.2
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.2
 	github.com/PagerDuty/go-pagerduty v1.8.0
 	github.com/broadinstitute/sherlock/go-shared v0.0.0
 	github.com/coreos/go-oidc v2.2.1+incompatible
@@ -18,6 +19,7 @@ require (
 	github.com/jackc/pgx/v5 v5.6.0
 	github.com/jinzhu/copier v0.4.0
 	github.com/knadh/koanf v1.5.0
+	github.com/microsoftgraph/msgraph-sdk-go v1.44.0
 	github.com/pact-foundation/pact-go/v2 v2.0.5
 	github.com/rs/zerolog v1.33.0
 	github.com/slack-go/slack v0.13.0
@@ -42,11 +44,15 @@ require (
 	cloud.google.com/go/auth/oauth2adapt v0.2.2 // indirect
 	cloud.google.com/go/compute/metadata v0.3.0 // indirect
 	filippo.io/edwards25519 v1.1.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
 	github.com/KyleBanks/depth v1.2.1 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bytedance/sonic v1.11.6 // indirect
 	github.com/bytedance/sonic/loader v0.1.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/cjlapao/common-go v0.0.39 // indirect
 	github.com/cloudwego/base64x v0.1.4 // indirect
 	github.com/cloudwego/iasm v0.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
@@ -67,6 +73,7 @@ require (
 	github.com/go-playground/validator/v10 v10.20.0 // indirect
 	github.com/go-sql-driver/mysql v1.8.1 // indirect
 	github.com/goccy/go-json v0.10.2 // indirect
+	github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
@@ -88,18 +95,28 @@ require (
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.7 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
+	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/leodido/go-urn v1.4.0 // indirect
 	github.com/lib/pq v1.10.9 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
+	github.com/microsoft/kiota-abstractions-go v1.6.0 // indirect
+	github.com/microsoft/kiota-authentication-azure-go v1.0.2 // indirect
+	github.com/microsoft/kiota-http-go v1.3.1 // indirect
+	github.com/microsoft/kiota-serialization-form-go v1.0.0 // indirect
+	github.com/microsoft/kiota-serialization-json-go v1.0.7 // indirect
+	github.com/microsoft/kiota-serialization-multipart-go v1.0.0 // indirect
+	github.com/microsoft/kiota-serialization-text-go v1.0.0 // indirect
+	github.com/microsoftgraph/msgraph-sdk-go-core v1.1.0 // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.2 // indirect
+	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021 // indirect
 	github.com/prometheus/client_golang v1.14.0 // indirect
@@ -111,6 +128,7 @@ require (
 	github.com/spf13/afero v1.11.0 // indirect
 	github.com/spf13/cobra v1.8.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/std-uritemplate/std-uritemplate/go v0.0.55 // indirect
 	github.com/stretchr/objx v0.5.2 // indirect
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
 	github.com/ugorji/go/codec v1.2.12 // indirect