Skip to content

Commit

Permalink
Add logic to check prefix of image url and set an imagePullSecret
Browse files Browse the repository at this point in the history
add instructions to install guide

ensure pull secret is reconciled when changed

Signed-off-by: craig <[email protected]>

rh-pre-commit.version: 2.2.0
rh-pre-commit.check-secrets: ENABLED
  • Loading branch information
maleck13 committed Dec 18, 2024
1 parent 6195492 commit 800770e
Show file tree
Hide file tree
Showing 6 changed files with 294 additions and 12 deletions.
5 changes: 5 additions & 0 deletions controllers/data_plane_policies_workflow.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,11 @@ const (

var (
WASMFilterImageURL = env.GetString("RELATED_IMAGE_WASMSHIM", "oci://quay.io/kuadrant/wasm-shim:latest")
// protectedRegistry this defines a default protected registry. If this is in the wasm image URL we add a pull secret name to the WASMPLugin resource
ProtectedRegistry = env.GetString("PROTECTED_REGISTRY", "registry.redhat.io")

// registryPullSecretName this is the pull secret name we will add to the WASMPlugin if the URL for he image is from the defined PROTECTED_REGISTRY
RegistryPullSecretName = "wasm-plugin-pull-secret"

StateIstioExtensionsModified = "IstioExtensionsModified"
StateEnvoyGatewayExtensionsModified = "EnvoyGatewayExtensionsModified"
Expand Down
24 changes: 18 additions & 6 deletions controllers/envoy_gateway_extension_reconciler.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
"strings"
"sync"

envoygatewayv1alpha1 "github.com/envoyproxy/gateway/api/v1alpha1"
Expand All @@ -15,7 +16,9 @@ import (
k8stypes "k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/dynamic"
"k8s.io/utils/ptr"
v1 "sigs.k8s.io/gateway-api/apis/v1"
gatewayapiv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2"
gwapiv1b1 "sigs.k8s.io/gateway-api/apis/v1beta1"

kuadrantv1 "github.com/kuadrant/kuadrant-operator/api/v1"
kuadrantv1beta1 "github.com/kuadrant/kuadrant-operator/api/v1beta1"
Expand Down Expand Up @@ -52,7 +55,7 @@ func (r *EnvoyGatewayExtensionReconciler) Subscription() controller.Subscription
func (r *EnvoyGatewayExtensionReconciler) Reconcile(ctx context.Context, _ []controller.ResourceEvent, topology *machinery.Topology, _ error, state *sync.Map) error {
logger := controller.LoggerFromContext(ctx).WithName("EnvoyGatewayExtensionReconciler")

logger.V(1).Info("building envoy gateway extension")
logger.V(1).Info("building envoy gateway extension", "image url", WASMFilterImageURL)
defer logger.V(1).Info("finished building envoy gateway extension")

// build wasm plugin configs for each gateway
Expand All @@ -76,8 +79,7 @@ func (r *EnvoyGatewayExtensionReconciler) Reconcile(ctx context.Context, _ []con

for _, gateway := range gateways {
gatewayKey := k8stypes.NamespacedName{Name: gateway.GetName(), Namespace: gateway.GetNamespace()}

desiredEnvoyExtensionPolicy := buildEnvoyExtensionPolicyForGateway(gateway, wasmConfigs[gateway.GetLocator()])
desiredEnvoyExtensionPolicy := buildEnvoyExtensionPolicyForGateway(gateway, wasmConfigs[gateway.GetLocator()], ProtectedRegistry, WASMFilterImageURL)

resource := r.client.Resource(kuadrantenvoygateway.EnvoyExtensionPoliciesResource).Namespace(desiredEnvoyExtensionPolicy.GetNamespace())

Expand Down Expand Up @@ -216,7 +218,7 @@ func (r *EnvoyGatewayExtensionReconciler) buildWasmConfigs(ctx context.Context,
}

// buildEnvoyExtensionPolicyForGateway builds a desired EnvoyExtensionPolicy custom resource for a given gateway and corresponding wasm config
func buildEnvoyExtensionPolicyForGateway(gateway *machinery.Gateway, wasmConfig wasm.Config) *envoygatewayv1alpha1.EnvoyExtensionPolicy {
func buildEnvoyExtensionPolicyForGateway(gateway *machinery.Gateway, wasmConfig wasm.Config, protectedRegistry, imageURL string) *envoygatewayv1alpha1.EnvoyExtensionPolicy {
envoyPolicy := &envoygatewayv1alpha1.EnvoyExtensionPolicy{
TypeMeta: metav1.TypeMeta{
Kind: kuadrantenvoygateway.EnvoyExtensionPolicyGroupKind.Kind,
Expand Down Expand Up @@ -256,7 +258,7 @@ func buildEnvoyExtensionPolicyForGateway(gateway *machinery.Gateway, wasmConfig
Code: envoygatewayv1alpha1.WasmCodeSource{
Type: envoygatewayv1alpha1.ImageWasmCodeSourceType,
Image: &envoygatewayv1alpha1.ImageWasmCodeSource{
URL: WASMFilterImageURL,
URL: imageURL,
},
},
Config: nil,
Expand All @@ -268,6 +270,16 @@ func buildEnvoyExtensionPolicyForGateway(gateway *machinery.Gateway, wasmConfig
},
},
}
for _, wasm := range envoyPolicy.Spec.Wasm {
if wasm.Code.Image.PullSecretRef != nil {
//reset it to empty this will remove it if the image is now public registry
wasm.Code.Image.PullSecretRef = nil
}

Check warning on line 277 in controllers/envoy_gateway_extension_reconciler.go

View check run for this annotation

Codecov / codecov/patch

controllers/envoy_gateway_extension_reconciler.go#L275-L277

Added lines #L275 - L277 were not covered by tests
// if we are in a protected registry set the object
if protectedRegistry != "" && strings.Contains(imageURL, protectedRegistry) {
wasm.Code.Image.PullSecretRef = &gwapiv1b1.SecretObjectReference{Name: v1.ObjectName(RegistryPullSecretName)}
}
}

if len(wasmConfig.ActionSets) == 0 {
utils.TagObjectToDelete(envoyPolicy)
Expand All @@ -292,7 +304,7 @@ func equalEnvoyExtensionPolicies(a, b *envoygatewayv1alpha1.EnvoyExtensionPolicy

return len(aWasms) == len(bWasms) && lo.EveryBy(aWasms, func(aWasm envoygatewayv1alpha1.Wasm) bool {
return lo.SomeBy(bWasms, func(bWasm envoygatewayv1alpha1.Wasm) bool {
if ptr.Deref(aWasm.Name, "") != ptr.Deref(bWasm.Name, "") || ptr.Deref(aWasm.RootID, "") != ptr.Deref(bWasm.RootID, "") || ptr.Deref(aWasm.FailOpen, false) != ptr.Deref(bWasm.FailOpen, false) || aWasm.Code.Type != bWasm.Code.Type || aWasm.Code.Image.URL != bWasm.Code.Image.URL {
if ptr.Deref(aWasm.Name, "") != ptr.Deref(bWasm.Name, "") || ptr.Deref(aWasm.RootID, "") != ptr.Deref(bWasm.RootID, "") || ptr.Deref(aWasm.FailOpen, false) != ptr.Deref(bWasm.FailOpen, false) || aWasm.Code.Type != bWasm.Code.Type || aWasm.Code.Image.URL != bWasm.Code.Image.URL || ptr.Deref(aWasm.Code.Image.PullSecretRef, gwapiv1b1.SecretObjectReference{}) != ptr.Deref(bWasm.Code.Image.PullSecretRef, gwapiv1b1.SecretObjectReference{}) {
return false
}
aConfig, err := wasm.ConfigFromJSON(aWasm.Config)
Expand Down
178 changes: 178 additions & 0 deletions controllers/extenstion_reconciler_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,178 @@
//go:build unit

package controllers

import (
"fmt"
"testing"

envoygatewayv1alpha1 "github.com/envoyproxy/gateway/api/v1alpha1"
"github.com/kuadrant/kuadrant-operator/pkg/wasm"
"github.com/kuadrant/policy-machinery/machinery"
istioclientgoextensionv1alpha1 "istio.io/client-go/pkg/apis/extensions/v1alpha1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
v1 "sigs.k8s.io/gateway-api/apis/v1"
)

var (
defaultWasmImage = WASMFilterImageURL
registry = "protected.registry.io"
protectedRegImage = fmt.Sprintf("oci://%s/kuadrant/wasm-shim:latest", registry)
testGateway = &machinery.Gateway{
Gateway: &v1.Gateway{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "test",
},
},
}
testWasmConfig = wasm.Config{
ActionSets: []wasm.ActionSet{
{
Name: "test",
},
},
}
)

func Test_buildIstioWasmPluginForGateway(t *testing.T) {
testCases := []struct {
Name string
WASMImageURLS func() []string
ProtectedRegistryPrefix string
Assert func(t *testing.T, plugin *istioclientgoextensionv1alpha1.WasmPlugin)
}{
{
Name: "ensure image pull secret is set in wasmPlugin for protected registry",
WASMImageURLS: func() []string {
// note currently this is a package global
return []string{protectedRegImage}
},
ProtectedRegistryPrefix: registry,
Assert: func(t *testing.T, plugin *istioclientgoextensionv1alpha1.WasmPlugin) {
if plugin == nil {
t.Fatalf("Expected a wasmplugin")
}
if plugin.Spec.ImagePullSecret != RegistryPullSecretName {
t.Fatalf("Expected wasm plugin to have imagePullSecret %s but got %s", RegistryPullSecretName, plugin.Spec.ImagePullSecret)
}
},
},
{
Name: "ensure image pull secret is NOT set in wasmPlugin for unprotected registry",
WASMImageURLS: func() []string {
return []string{WASMFilterImageURL}
},
Assert: func(t *testing.T, plugin *istioclientgoextensionv1alpha1.WasmPlugin) {
if plugin == nil {
t.Fatalf("Expected a wasmplugin")
}
if plugin.Spec.ImagePullSecret != "" {
t.Fatalf("Expected wasm plugin to NOT have imagePullSecret %v", plugin.Spec.ImagePullSecret)
}
},
},
{
Name: "ensure image pull secret is set in wasmPlugin for protected registry and unset for unprotected registry",
WASMImageURLS: func() []string {
return []string{ProtectedRegistry, WASMFilterImageURL}
},
Assert: func(t *testing.T, plugin *istioclientgoextensionv1alpha1.WasmPlugin) {
if plugin == nil {
t.Fatalf("Expected a wasmplugin")
}
if plugin.Spec.Url == protectedRegImage && plugin.Spec.ImagePullSecret == "" {
t.Fatalf("Expected wasm plugin to have imagePullSecret set but got none")
}
if plugin.Spec.Url == WASMFilterImageURL && plugin.Spec.ImagePullSecret != "" {
t.Fatalf("Expected wasm plugin to not have imagePullSecret set but got %v", plugin.Spec.ImagePullSecret)
}
},
},
}

for _, testCase := range testCases {
t.Run(testCase.Name, func(t *testing.T) {
images := testCase.WASMImageURLS()
for _, image := range images {
plugin := buildIstioWasmPluginForGateway(testGateway, testWasmConfig, testCase.ProtectedRegistryPrefix, image)
testCase.Assert(t, plugin)
}
})
}

}

func Test_buildEnvoyExtensionPolicyForGateway(t *testing.T) {
testCases := []struct {
Name string
WASMImageURLS func() []string
ProtectedRegistryPrefix string
Assert func(t *testing.T, policy *envoygatewayv1alpha1.EnvoyExtensionPolicy)
}{
{
Name: "ensure image pull secret is set in ExtensionPolicy for protected registry",
WASMImageURLS: func() []string {
return []string{protectedRegImage}
},
ProtectedRegistryPrefix: registry,
Assert: func(t *testing.T, policy *envoygatewayv1alpha1.EnvoyExtensionPolicy) {
if policy == nil {
t.Fatalf("Expected a wasmplugin")
}
for _, w := range policy.Spec.Wasm {
if w.Code.Image.PullSecretRef == nil {
t.Fatalf("Expected extension to have imagePullSecret %v but no pullSecretRef", RegistryPullSecretName)
}
if w.Code.Image.PullSecretRef.Name != v1.ObjectName(RegistryPullSecretName) {
t.Fatalf("expected the pull secret name to be %s but got %v", RegistryPullSecretName, w.Code.Image.PullSecretRef.Name)
}
}
},
},
{
Name: "ensure image pull secret is NOT set in wasmPlugin for unprotected registry",
WASMImageURLS: func() []string {
return []string{defaultWasmImage}
},
Assert: func(t *testing.T, policy *envoygatewayv1alpha1.EnvoyExtensionPolicy) {
if policy == nil {
t.Fatalf("Expected a wasmplugin")
}
for _, w := range policy.Spec.Wasm {
if w.Code.Image.PullSecretRef != nil {
t.Fatalf("Expected extension to have not imagePullSecret but got %v", w.Code.Image.PullSecretRef)
}
}
},
},
{
Name: "ensure image pull secret is set in extension for protected registry and unset for unprotected registry",
WASMImageURLS: func() []string {
return []string{ProtectedRegistry, WASMFilterImageURL}
},
Assert: func(t *testing.T, policy *envoygatewayv1alpha1.EnvoyExtensionPolicy) {
if policy == nil {
t.Fatalf("Expected a wasmplugin")
}
for _, w := range policy.Spec.Wasm {
if w.Code.Image.PullSecretRef == nil && w.Code.Image.URL == protectedRegImage {
t.Fatalf("Expected policy to have imagePullSecret set but got none")
}
if w.Code.Image.PullSecretRef != nil && w.Code.Image.URL == WASMFilterImageURL {
t.Fatalf("Expected policy to not have imagePullSecret set but got %v", w.Code.Image.PullSecretRef)
}
}

},
},
}

for _, testCase := range testCases {
images := testCase.WASMImageURLS()
for _, image := range images {
policy := buildEnvoyExtensionPolicyForGateway(testGateway, testWasmConfig, testCase.ProtectedRegistryPrefix, image)
testCase.Assert(t, policy)
}
}
}
20 changes: 14 additions & 6 deletions controllers/istio_extension_reconciler.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
"strings"
"sync"

"github.com/kuadrant/policy-machinery/controller"
Expand Down Expand Up @@ -53,7 +54,7 @@ func (r *IstioExtensionReconciler) Subscription() controller.Subscription {
func (r *IstioExtensionReconciler) Reconcile(ctx context.Context, _ []controller.ResourceEvent, topology *machinery.Topology, _ error, state *sync.Map) error {
logger := controller.LoggerFromContext(ctx).WithName("IstioExtensionReconciler")

logger.V(1).Info("building istio extension")
logger.V(1).Info("building istio extension ", "image url", WASMFilterImageURL)
defer logger.V(1).Info("finished building istio extension")

// build wasm plugin configs for each gateway
Expand All @@ -78,7 +79,7 @@ func (r *IstioExtensionReconciler) Reconcile(ctx context.Context, _ []controller
for _, gateway := range gateways {
gatewayKey := k8stypes.NamespacedName{Name: gateway.GetName(), Namespace: gateway.GetNamespace()}

desiredWasmPlugin := buildIstioWasmPluginForGateway(gateway, wasmConfigs[gateway.GetLocator()])
desiredWasmPlugin := buildIstioWasmPluginForGateway(gateway, wasmConfigs[gateway.GetLocator()], ProtectedRegistry, WASMFilterImageURL)

resource := r.client.Resource(kuadrantistio.WasmPluginsResource).Namespace(desiredWasmPlugin.GetNamespace())

Expand Down Expand Up @@ -114,7 +115,7 @@ func (r *IstioExtensionReconciler) Reconcile(ctx context.Context, _ []controller
}
continue
}

logger.V(1).Info("wasmplugin object ", "desired", desiredWasmPlugin)
if equalWasmPlugins(existingWasmPlugin, desiredWasmPlugin) {
logger.V(1).Info("wasmplugin object is up to date, nothing to do")
continue
Expand All @@ -125,6 +126,7 @@ func (r *IstioExtensionReconciler) Reconcile(ctx context.Context, _ []controller
existingWasmPlugin.Spec.Phase = desiredWasmPlugin.Spec.Phase
existingWasmPlugin.Spec.TargetRefs = desiredWasmPlugin.Spec.TargetRefs
existingWasmPlugin.Spec.PluginConfig = desiredWasmPlugin.Spec.PluginConfig
existingWasmPlugin.Spec.ImagePullSecret = desiredWasmPlugin.Spec.ImagePullSecret

existingWasmPluginUnstructured, err := controller.Destruct(existingWasmPlugin)
if err != nil {
Expand Down Expand Up @@ -228,7 +230,7 @@ func hasAuthAccess(actionSet []wasm.Action) bool {
}

// buildIstioWasmPluginForGateway builds a desired WasmPlugin custom resource for a given gateway and corresponding wasm config
func buildIstioWasmPluginForGateway(gateway *machinery.Gateway, wasmConfig wasm.Config) *istioclientgoextensionv1alpha1.WasmPlugin {
func buildIstioWasmPluginForGateway(gateway *machinery.Gateway, wasmConfig wasm.Config, protectedRegistry, imageURL string) *istioclientgoextensionv1alpha1.WasmPlugin {
wasmPlugin := &istioclientgoextensionv1alpha1.WasmPlugin{
TypeMeta: metav1.TypeMeta{
Kind: kuadrantistio.WasmPluginGroupKind.Kind,
Expand Down Expand Up @@ -257,11 +259,17 @@ func buildIstioWasmPluginForGateway(gateway *machinery.Gateway, wasmConfig wasm.
Name: gateway.GetName(),
},
},
Url: WASMFilterImageURL,
Url: imageURL,
PluginConfig: nil,
Phase: istioextensionsv1alpha1.PluginPhase_STATS, // insert the plugin before Istio stats filters and after Istio authorization filters.
},
}
// reset to empty to allow fo the image having moved to a public registry
wasmPlugin.Spec.ImagePullSecret = ""
// only set to pull secret if we are in a protected registry
if protectedRegistry != "" && strings.Contains(imageURL, protectedRegistry) {
wasmPlugin.Spec.ImagePullSecret = RegistryPullSecretName
}

if len(wasmConfig.ActionSets) == 0 {
utils.TagObjectToDelete(wasmPlugin)
Expand All @@ -277,7 +285,7 @@ func buildIstioWasmPluginForGateway(gateway *machinery.Gateway, wasmConfig wasm.
}

func equalWasmPlugins(a, b *istioclientgoextensionv1alpha1.WasmPlugin) bool {
if a.Spec.Url != b.Spec.Url || a.Spec.Phase != b.Spec.Phase || !kuadrantistio.EqualTargetRefs(a.Spec.TargetRefs, b.Spec.TargetRefs) {
if a.Spec.ImagePullSecret != b.Spec.ImagePullSecret || a.Spec.Url != b.Spec.Url || a.Spec.Phase != b.Spec.Phase || !kuadrantistio.EqualTargetRefs(a.Spec.TargetRefs, b.Spec.TargetRefs) {
return false
}

Expand Down
13 changes: 13 additions & 0 deletions doc/install/install-openshift.md
Original file line number Diff line number Diff line change
Expand Up @@ -288,6 +288,19 @@ spec:
upgradeStrategy: Default
EOF
```
**Authenticated Registry**

!!! note

If you need to use a wasm image from a protected registry (such as the redhat registry), you will need to configure an image pull secret to use. This secret must exist in each namespace where there is a Gateway resource defined that you intend to target with `AuthPolicy` or `RatelimitPolicy` and it must be named `wasm-plugin-pull-secret`.

Example Creating Pull Secret:

```bash
kubectl create secret docker-registry wasm-plugin-pull-secret -n ${GATEWAY_NAMESPACE} \ --docker-server=my.registry.io \ --docker-username=your-registry-service-account-username \ --docker-password=your-registry-service-account-password
```

The configuration for the gateway will expect this secret to exist if the registry name begins with `registry.redhat.com` by default. This can be changed via the env var `PROTECTED_REGISTRY` set in the kuadrant-operator.

Wait for the Kuadrant Operators to be installed as follows:

Expand Down
Loading

0 comments on commit 800770e

Please sign in to comment.