Skip to content

Commit

Permalink
Teleport Matchine ID & Tests (#37)
Browse files Browse the repository at this point in the history 
* Refactored controller to support tbot credentials & added tests

* Reordered imports

* Improvements in tests

* Updated nancyignore

* Added configmap for tbot

* Added tbot rbac

* Fixed tbot charts

* Added a delay to requeue in case i(re-)connnection to Teleport fails

* Added requeue delay on failed identity renewal

* Adjusted tests

* Improved loging of identity age

* Removed rescheduling of failed teleport identity renewal

* Cleaned up Helm charts

* Fixed controller test

* Cleaned up unused Helm charts

* Adjusted tbot configmap

* Cleaned up
  • Loading branch information
@vvondruska
vvondruska authored Nov 16, 2023
1 parent 6fbefce commit 7a5f735
Show file tree
Hide file tree
Showing 20 changed files with 529 additions and 297 deletions.
1 change: 1 addition & 0 deletions .nancy-ignore
View file
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
#pkg:golang/k8s.io/[email protected]
CVE-2020-8561 until=2024-01-08 # k8s.io/[email protected]
CVE-2023-47108 until=2024-01-08 # go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/[email protected]
32 changes: 10 additions & 22 deletions helm/teleport-operator/templates/configmap.yaml
View file
Original file line number Diff line number Diff line change
@@ -1,27 +1,15 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: tbot-config
namespace: {{ include "resource.default.namespace" . }}
name: {{ include "resource.default.name" . }}
namespace: {{ include "resource.default.namespace" . }}
labels:
{{- include "labels.common" . | nindent 4 }}
{{- include "labels.common" . | nindent 4 }}
type: Opaque
data:
tbot.yaml: |
version: v2
onboarding:
join_method: kubernetes
# ensure token is set to the name of the join token you created earlier
token: {{ .Values.teleport.managementClusterName }}
storage:
# a memory destination is used for the bots own state since the kubernetes
# join method does not require persistence.
type: memory
# ensure this is configured to the address of your Teleport Proxy or
# Auth Server. Prefer the address of the Teleport Proxy.
auth_server: {{ .Values.teleport.proxyAddr }}
# outputs will be filled in during the completion of an access guide.
outputs:
- type: identity
destination:
type: kubernetes_secret
name: identity-output
appCatalog: {{ .Values.teleport.appCatalog | quote }}
appName: {{ .Values.teleport.appName | quote }}
appVersion: {{ .Values.teleport.appVersion | quote }}
managementClusterName: {{ .Values.teleport.managementClusterName | quote }}
proxyAddr: {{ .Values.teleport.proxyAddr | quote }}
teleportVersion: {{ .Values.teleport.teleportVersion | quote }}
16 changes: 0 additions & 16 deletions helm/teleport-operator/templates/secret.yaml
View file

This file was deleted.

27 changes: 27 additions & 0 deletions helm/teleport-operator/templates/tbot-configmap.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: tbot-config
namespace: {{ include "resource.default.namespace" . }}
labels:
{{- include "labels.common" . | nindent 4 }}
data:
tbot.yaml: |
version: v2
onboarding:
join_method: kubernetes
# ensure token is set to the name of the join token you created earlier
token: {{ .Values.teleport.managementClusterName }}-bot
storage:
# a memory destination is used for the bots own state since the kubernetes
# join method does not require persistence.
type: memory
# ensure this is configured to the address of your Teleport Proxy or
# Auth Server. Prefer the address of the Teleport Proxy.
auth_server: {{ .Values.teleport.proxyAddr }}
# outputs will be filled in during the completion of an access guide.
outputs:
- type: identity
destination:
type: kubernetes_secret
name: identity-output
36 changes: 17 additions & 19 deletions internal/controller/cluster_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,6 @@ package controller

import (
"context"
"crypto/sha512"
"encoding/hex"
"time"

"github.com/giantswarm/microerror"
Expand All @@ -29,13 +27,16 @@ import (
capi "sigs.k8s.io/cluster-api/api/v1beta1"
ctrl "sigs.k8s.io/controller-runtime"

"github.com/giantswarm/teleport-operator/internal/pkg/config"
"github.com/giantswarm/teleport-operator/internal/pkg/key"
"github.com/giantswarm/teleport-operator/internal/pkg/teleport"

"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
)

const identityExpirationPeriod = 20 * time.Minute

// ClusterReconciler reconciles a Cluster object
type ClusterReconciler struct {
Client client.Client
Expand Down Expand Up @@ -71,35 +72,32 @@ func (r *ClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
}
log.Info("Reconciling cluster", "cluster", cluster)

now := time.Now()
diff := now.Sub(r.Teleport.SecretConfig.LastRead)
seconds := diff.Seconds()
minutes := seconds / 60
hasher := sha512.New()
hasher.Write([]byte(r.Teleport.SecretConfig.IdentityFile))
sum := hasher.Sum(nil)
hashString := hex.EncodeToString(sum)

log.Info("Teleport identity", "last-read-minutes-ago", minutes, "hash", hashString)
if r.Teleport.Identity != nil {
log.Info("Teleport identity", "last-read-minutes-ago", r.Teleport.Identity.Age(), "hash", r.Teleport.Identity.Hash())
}

if time.Since(r.Teleport.SecretConfig.LastRead) > 20*time.Minute {
if r.Teleport.Identity == nil || time.Since(r.Teleport.Identity.LastRead) > identityExpirationPeriod {
log.Info("Retrieving new identity", "secretName", key.TeleportBotSecretName)

newSecretConfig, err := teleport.GetConfigFromSecret(ctx, r.Client, r.Namespace)
newIdentityConfig, err := config.GetIdentityConfigFromSecret(ctx, r.Client, r.Namespace)
if err != nil {
return ctrl.Result{}, microerror.Mask(err)
}
r.Teleport.SecretConfig = newSecretConfig

if r.Teleport.TeleportClient, err = teleport.NewClient(ctx, newSecretConfig.ProxyAddr, newSecretConfig.IdentityFile); err != nil {
if r.Teleport.TeleportClient, err = teleport.NewClient(ctx, r.Teleport.Config.ProxyAddr, newIdentityConfig.IdentityFile); err != nil {
return ctrl.Result{}, microerror.Mask(err)
}
log.Info("Re-connected to teleport cluster with new identity", "proxyAddr", newSecretConfig.ProxyAddr)
if r.Teleport.Identity == nil {
log.Info("Connected to teleport cluster", "proxyAddr", r.Teleport.Config.ProxyAddr)
} else {
log.Info("Re-connected to teleport cluster with new identity", "proxyAddr", r.Teleport.Config.ProxyAddr)
}
r.Teleport.Identity = newIdentityConfig
}

registerName := cluster.Name
if cluster.Name != r.Teleport.SecretConfig.ManagementClusterName {
registerName = key.GetRegisterName(r.Teleport.SecretConfig.ManagementClusterName, cluster.Name)
if cluster.Name != r.Teleport.Config.ManagementClusterName {
registerName = key.GetRegisterName(r.Teleport.Config.ManagementClusterName, cluster.Name)
}

// Check if the cluster instance is marked to be deleted, which is indicated by the deletion timestamp being set.
Expand Down
121 changes: 100 additions & 21 deletions internal/controller/cluster_controller_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package controller

import (
"context"
"errors"
"testing"
"time"

Expand All @@ -15,6 +16,7 @@ import (
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"

"github.com/giantswarm/teleport-operator/internal/pkg/config"
"github.com/giantswarm/teleport-operator/internal/pkg/key"
"github.com/giantswarm/teleport-operator/internal/pkg/teleport"
"github.com/giantswarm/teleport-operator/internal/pkg/test"
Expand All @@ -26,19 +28,24 @@ func Test_ClusterController(t *testing.T) {
namespace string
token string
tokens []teleportTypes.ProvisionToken
secretConfig *teleport.SecretConfig
config *config.Config
identity *config.IdentityConfig
identitySecret *corev1.Secret
cluster *capi.Cluster
secret *corev1.Secret
configMap *corev1.ConfigMap
newTeleportClient func(ctx context.Context, proxyAddr, identityFile string) (teleport.Client, error)
expectedCluster *capi.Cluster
expectedSecret *corev1.Secret
expectedConfigMap *corev1.ConfigMap
expectedError error
}{
{
name: "case 0: Register cluster and create Secret, ConfigMap and App resources in case they do not exist",
namespace: test.NamespaceName,
token: test.TokenName,
secretConfig: newSecretConfig(),
config: newConfig(),
identity: newIdentity(test.LastReadValue),
cluster: test.NewCluster(test.ClusterName, test.NamespaceName, []string{key.TeleportOperatorFinalizer}, time.Time{}),
expectedCluster: test.NewCluster(test.ClusterName, test.NamespaceName, []string{key.TeleportOperatorFinalizer}, time.Time{}),
expectedSecret: test.NewSecret(test.ClusterName, test.NamespaceName, test.TokenName),
Expand All @@ -48,7 +55,8 @@ func Test_ClusterController(t *testing.T) {
name: "case 1: Register cluster and update Secret, ConfigMap and App resources in case they exist",
namespace: test.NamespaceName,
token: test.TokenName,
secretConfig: newSecretConfig(),
config: newConfig(),
identity: newIdentity(test.LastReadValue),
cluster: test.NewCluster(test.ClusterName, test.NamespaceName, []string{key.TeleportOperatorFinalizer}, time.Time{}),
secret: test.NewSecret(test.ClusterName, test.NamespaceName, test.TokenName),
configMap: test.NewConfigMap(test.ClusterName, test.AppName, test.NamespaceName, test.TokenName),
Expand All @@ -60,7 +68,8 @@ func Test_ClusterController(t *testing.T) {
name: "case 2: Update Secret and ConfigMap resources in case join token changes",
namespace: test.NamespaceName,
token: test.NewTokenName,
secretConfig: newSecretConfig(),
config: newConfig(),
identity: newIdentity(test.LastReadValue),
cluster: test.NewCluster(test.ClusterName, test.NamespaceName, []string{key.TeleportOperatorFinalizer}, time.Time{}),
secret: test.NewSecret(test.ClusterName, test.NamespaceName, test.TokenName),
configMap: test.NewConfigMap(test.ClusterName, test.AppName, test.NamespaceName, test.TokenName),
Expand All @@ -69,13 +78,45 @@ func Test_ClusterController(t *testing.T) {
expectedConfigMap: test.NewConfigMap(test.ClusterName, test.AppName, test.NamespaceName, test.NewTokenName),
},
{
name: "case 3: Deregister cluster and delete resources in case the cluster is deleted",
namespace: test.NamespaceName,
token: test.TokenName,
secretConfig: newSecretConfig(),
cluster: test.NewCluster(test.ClusterName, test.NamespaceName, []string{key.TeleportOperatorFinalizer}, time.Now()),
secret: test.NewSecret(test.ClusterName, test.NamespaceName, test.TokenName),
configMap: test.NewConfigMap(test.ClusterName, test.AppName, test.NamespaceName, test.TokenName),
name: "case 3: Deregister cluster and delete resources in case the cluster is deleted",
namespace: test.NamespaceName,
token: test.TokenName,
config: newConfig(),
identity: newIdentity(test.LastReadValue),
cluster: test.NewCluster(test.ClusterName, test.NamespaceName, []string{key.TeleportOperatorFinalizer}, time.Now()),
secret: test.NewSecret(test.ClusterName, test.NamespaceName, test.TokenName),
configMap: test.NewConfigMap(test.ClusterName, test.AppName, test.NamespaceName, test.TokenName),
},
{
name: "case 4: Reconnect to Teleport when credentials are rotated",
namespace: test.NamespaceName,
token: test.NewTokenName,
config: newConfig(),
identity: newIdentity(time.Now().Add(-identityExpirationPeriod - time.Second)),
cluster: test.NewCluster(test.ClusterName, test.NamespaceName, []string{key.TeleportOperatorFinalizer}, time.Time{}),
secret: test.NewSecret(test.ClusterName, test.NamespaceName, test.TokenName),
identitySecret: test.NewIdentitySecret(test.NamespaceName, test.IdentityFileValue),
configMap: test.NewConfigMap(test.ClusterName, test.AppName, test.NamespaceName, test.TokenName),
newTeleportClient: func(ctx context.Context, proxyAddr, identityFile string) (teleport.Client, error) {
return test.NewTeleportClient(test.FakeTeleportClientConfig{Tokens: nil}), nil
},
expectedCluster: test.NewCluster(test.ClusterName, test.NamespaceName, []string{key.TeleportOperatorFinalizer}, time.Time{}),
expectedSecret: test.NewSecret(test.ClusterName, test.NamespaceName, test.NewTokenName),
expectedConfigMap: test.NewConfigMap(test.ClusterName, test.AppName, test.NamespaceName, test.NewTokenName),
},
{
name: "case 5: Return an error in case reconnection to Teleport fails after the credentials are rotated",
namespace: test.NamespaceName,
token: test.TokenName,
config: newConfig(),
identity: newIdentity(time.Now().Add(-identityExpirationPeriod - time.Second)),
cluster: test.NewCluster(test.ClusterName, test.NamespaceName, []string{key.TeleportOperatorFinalizer}, time.Time{}),
secret: test.NewSecret(test.ClusterName, test.NamespaceName, test.TokenName),
configMap: test.NewConfigMap(test.ClusterName, test.AppName, test.NamespaceName, test.TokenName),
newTeleportClient: func(ctx context.Context, proxyAddr, identityFile string) (teleport.Client, error) {
return nil, errors.New("simulated error")
},
expectedError: errors.New("secrets \"identity-output\" not found"),
},
}

Expand All @@ -91,6 +132,17 @@ func Test_ClusterController(t *testing.T) {
if tc.configMap != nil {
runtimeObjects = append(runtimeObjects, tc.configMap)
}
if tc.identitySecret != nil {
runtimeObjects = append(runtimeObjects, tc.identitySecret)
}

newTeleportClient := teleport.NewClient
if tc.newTeleportClient != nil {
teleport.NewClient = tc.newTeleportClient
}
defer func() {
teleport.NewClient = newTeleportClient
}()

ctrlClient, err := test.NewFakeK8sClient(runtimeObjects)
if err != nil {
Expand All @@ -105,11 +157,12 @@ func Test_ClusterController(t *testing.T) {
Log: log,
Scheme: scheme.Scheme,
Namespace: tc.namespace,
Teleport: teleport.New(tc.namespace, tc.secretConfig, test.NewMockTokenGenerator(tc.token)),
Teleport: teleport.New(tc.namespace, tc.config, test.NewMockTokenGenerator(tc.token)),
}
controller.Teleport.TeleportClient = test.NewTeleportClient(test.FakeTeleportClientConfig{
Tokens: tc.tokens,
})
controller.Teleport.Identity = tc.identity

req := ctrl.Request{
NamespacedName: types.NamespacedName{
Expand All @@ -120,7 +173,14 @@ func Test_ClusterController(t *testing.T) {

_, err = controller.Reconcile(ctx, req)
if err != nil {
if tc.expectedError != nil && err.Error() != tc.expectedError.Error() {
t.Fatalf("unexpected error: expected %v, actual %v", tc.expectedError, err)
} else if tc.expectedError != nil {
return
}
t.Fatalf("unexpected error %v", err)
} else if tc.expectedError != nil {
t.Fatalf("did not receive an expected error %v", tc.expectedError)
}

clusterList := &capi.ClusterList{}
Expand All @@ -142,12 +202,17 @@ func Test_ClusterController(t *testing.T) {
if err != nil {
t.Fatalf("unexpected error %v", err)
}
expectedSecretListLength := 1
if tc.identitySecret != nil {
expectedSecretListLength = 2
}
if tc.expectedSecret != nil {
if len(secretList.Items) == 0 {
if len(secretList.Items) < expectedSecretListLength {
t.Fatalf("unexpected result: secret list is empty\n%v", secretList)
}
test.CheckSecret(t, tc.expectedSecret, &secretList.Items[0])
} else if len(secretList.Items) > 0 {
actualSecret := findSecretInList(secretList, tc.expectedSecret.Name)
test.CheckSecret(t, tc.expectedSecret, actualSecret)
} else if len(secretList.Items) > expectedSecretListLength-1 {
t.Fatalf("unexpected result: secret list is not empty\n%v", secretList)
}

Expand All @@ -158,11 +223,11 @@ func Test_ClusterController(t *testing.T) {
}
if tc.expectedConfigMap != nil {
if len(configMapList.Items) == 0 {
t.Fatalf("unexpected result: secret list is empty\n%v", secretList)
t.Fatalf("unexpected result: config map list is empty\n%v", configMapList)
}
test.CheckConfigMap(t, tc.expectedConfigMap, &configMapList.Items[0])
} else if len(configMapList.Items) > 0 {
t.Fatalf("unexpected result: config map list is not empty\n%v", secretList)
t.Fatalf("unexpected result: config map list is not empty\n%v", configMapList)
}

appList := &appv1alpha1.AppList{}
Expand All @@ -174,15 +239,29 @@ func Test_ClusterController(t *testing.T) {
}
}

func newSecretConfig() *teleport.SecretConfig {
return &teleport.SecretConfig{
func newConfig() *config.Config {
return &config.Config{
AppCatalog: test.AppCatalog,
AppName: test.AppName,
AppVersion: test.AppVersion,
IdentityFile: test.IdentityFileValue,
LastRead: test.LastReadValue,
ManagementClusterName: test.ManagementClusterName,
ProxyAddr: test.ProxyAddr,
TeleportVersion: test.TeleportVersion,
}
}

func newIdentity(lastRead time.Time) *config.IdentityConfig {
return &config.IdentityConfig{
IdentityFile: test.IdentityFileValue,
LastRead: lastRead,
}
}

func findSecretInList(secretList *corev1.SecretList, name string) *corev1.Secret {
for _, secret := range secretList.Items {
if secret.Name == name {
return &secret
}
}
return nil
}
Loading

0 comments on commit 7a5f735

Please sign in to comment.