diff --git a/func/internal/podevaluator.go b/func/internal/podevaluator.go index 43a8d773..e4b7368b 100644 --- a/func/internal/podevaluator.go +++ b/func/internal/podevaluator.go @@ -16,9 +16,12 @@ package internal import ( "context" + "crypto/tls" + "crypto/x509" "encoding/json" "fmt" "net" + "net/http" "os" "path/filepath" "strconv" @@ -28,6 +31,7 @@ import ( "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" + containerregistry "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/nephio-project/porch/func/evaluator" util "github.com/nephio-project/porch/pkg/util" @@ -71,7 +75,20 @@ type podEvaluator struct { var _ Evaluator = &podEvaluator{} -func NewPodEvaluator(namespace, wrapperServerImage string, interval, ttl time.Duration, podTTLConfig string, functionPodTemplateName string, enablePrivateRegistries bool, registryAuthSecretPath string, registryAuthSecretName string) (Evaluator, error) { +func NewPodEvaluator( + namespace, + wrapperServerImage string, + interval, + ttl time.Duration, + podTTLConfig string, + functionPodTemplateName string, + enablePrivateRegistries bool, + registryAuthSecretPath string, + registryAuthSecretName string, + enablePrivateRegistriesTls bool, + tlsSecretPath string, +) (Evaluator, error) { + restCfg, err := config.GetConfig() if err != nil { return nil, fmt.Errorf("failed to get rest config: %w", err) @@ -100,15 +117,17 @@ func NewPodEvaluator(namespace, wrapperServerImage string, interval, ttl time.Du pe := &podEvaluator{ requestCh: reqCh, podCacheManager: &podCacheManager{ - gcScanInternal: interval, - podTTL: ttl, - enablePrivateRegistries: enablePrivateRegistries, - registryAuthSecretPath: registryAuthSecretPath, - registryAuthSecretName: registryAuthSecretName, - requestCh: reqCh, - podReadyCh: readyCh, - cache: map[string]*podAndGRPCClient{}, - waitlists: map[string][]chan<- *clientConnAndError{}, + gcScanInternal: interval, + podTTL: ttl, + enablePrivateRegistries: enablePrivateRegistries, + registryAuthSecretPath: registryAuthSecretPath, + registryAuthSecretName: registryAuthSecretName, + enablePrivateRegistriesTls: enablePrivateRegistriesTls, + tlsSecretPath: tlsSecretPath, + requestCh: reqCh, + podReadyCh: readyCh, + cache: map[string]*podAndGRPCClient{}, + waitlists: map[string][]chan<- *clientConnAndError{}, podManager: &podManager{ kubeClient: cl, @@ -177,6 +196,9 @@ type podCacheManager struct { registryAuthSecretPath string registryAuthSecretName string + enablePrivateRegistriesTls bool + tlsSecretPath string + // requestCh is a receive-only channel to receive requestCh <-chan *clientConnRequest // podReadyCh is a channel to receive the information when a pod is ready. @@ -245,7 +267,7 @@ func (pcm *podCacheManager) warmupCache(podTTLConfig string) error { // We invoke the function with useGenerateName=false so that the pod name is fixed, // since we want to ensure only one pod is created for each function. - pcm.podManager.getFuncEvalPodClient(ctx, fnImage, ttl, false, pcm.enablePrivateRegistries, pcm.registryAuthSecretPath, pcm.registryAuthSecretName) + pcm.podManager.getFuncEvalPodClient(ctx, fnImage, ttl, false, pcm.enablePrivateRegistries, pcm.registryAuthSecretPath, pcm.registryAuthSecretName, pcm.enablePrivateRegistriesTls, pcm.tlsSecretPath) klog.Infof("preloaded pod cache for function %v", fnImage) }) @@ -313,7 +335,7 @@ func (pcm *podCacheManager) podCacheManager() { pcm.waitlists[req.image] = append(list, req.grpcClientCh) // We invoke the function with useGenerateName=true to avoid potential name collision, since if pod foo is // being deleted and we can't use the same name. - go pcm.podManager.getFuncEvalPodClient(context.Background(), req.image, pcm.podTTL, true, pcm.enablePrivateRegistries, pcm.registryAuthSecretPath, pcm.registryAuthSecretName) + go pcm.podManager.getFuncEvalPodClient(context.Background(), req.image, pcm.podTTL, true, pcm.enablePrivateRegistries, pcm.registryAuthSecretPath, pcm.registryAuthSecretName, pcm.enablePrivateRegistriesTls, pcm.tlsSecretPath) case resp := <-pcm.podReadyCh: if resp.err != nil { klog.Warningf("received error from the pod manager: %v", resp.err) @@ -445,9 +467,9 @@ type digestAndEntrypoint struct { // time-to-live period for the pod. If useGenerateName is false, it will try to // create a pod with a fixed name. Otherwise, it will create a pod and let the // apiserver to generate the name from a template. -func (pm *podManager) getFuncEvalPodClient(ctx context.Context, image string, ttl time.Duration, useGenerateName bool, enablePrivateRegistries bool, registryAuthSecretPath string, registryAuthSecretName string) { +func (pm *podManager) getFuncEvalPodClient(ctx context.Context, image string, ttl time.Duration, useGenerateName bool, enablePrivateRegistries bool, registryAuthSecretPath string, registryAuthSecretName string, enablePrivateRegistriesTls bool, tlsSecretPath string) { c, err := func() (*podAndGRPCClient, error) { - podKey, err := pm.retrieveOrCreatePod(ctx, image, ttl, useGenerateName, enablePrivateRegistries, registryAuthSecretPath, registryAuthSecretName) + podKey, err := pm.retrieveOrCreatePod(ctx, image, ttl, useGenerateName, enablePrivateRegistries, registryAuthSecretPath, registryAuthSecretName, enablePrivateRegistriesTls, tlsSecretPath) if err != nil { return nil, err } @@ -539,7 +561,7 @@ type DockerConfig struct { } // imageDigestAndEntrypoint gets the entrypoint of a container image by looking at its metadata. -func (pm *podManager) imageDigestAndEntrypoint(ctx context.Context, image string, enablePrivateRegistries bool, registryAuthSecretPath string, registryAuthSecretName string) (*digestAndEntrypoint, error) { +func (pm *podManager) imageDigestAndEntrypoint(ctx context.Context, image string, enablePrivateRegistries bool, registryAuthSecretPath string, registryAuthSecretName string, enablePrivateRegistriesTls bool, tlsSecretPath string) (*digestAndEntrypoint, error) { start := time.Now() defer func() { klog.Infof("getting image metadata for %v took %v", image, time.Since(start)) @@ -569,7 +591,7 @@ func (pm *podManager) imageDigestAndEntrypoint(ctx context.Context, image string } } - return pm.getImageMetadata(ctx, ref, auth, image) + return pm.getImageMetadata(ctx, ref, auth, image, enablePrivateRegistries, enablePrivateRegistriesTls, tlsSecretPath) } // ensureCustomAuthSecret ensures that, if an image from a custom registry is requested, the appropriate credentials are passed into a secret for function pods to use when pulling. If the secret does not already exist, it is created. @@ -590,7 +612,7 @@ func (pm *podManager) getCustomAuth(ref name.Reference, registryAuthSecretPath s var dockerConfig DockerConfig if err := json.Unmarshal(dockerConfigBytes, &dockerConfig); err != nil { - klog.Errorf("error unmarshalling authentication file %v", err) + klog.Errorf("error unmarshaling authentication file %v", err) return nil, err } @@ -598,8 +620,8 @@ func (pm *podManager) getCustomAuth(ref name.Reference, registryAuthSecretPath s } // getImageMetadata retrieves the image digest and entrypoint. -func (pm *podManager) getImageMetadata(ctx context.Context, ref name.Reference, auth authn.Authenticator, image string) (*digestAndEntrypoint, error) { - img, err := remote.Image(ref, remote.WithAuth(auth), remote.WithContext(ctx)) +func (pm *podManager) getImageMetadata(ctx context.Context, ref name.Reference, auth authn.Authenticator, image string, enablePrivateRegistries bool, enablePrivateRegistriesTls bool, tlsSecretPath string) (*digestAndEntrypoint, error) { + img, err := getImage(ctx, ref, auth, image, enablePrivateRegistries, enablePrivateRegistriesTls, tlsSecretPath) if err != nil { return nil, err } @@ -625,15 +647,74 @@ func (pm *podManager) getImageMetadata(ctx context.Context, ref name.Reference, return de, nil } +func getImage(ctx context.Context, ref name.Reference, auth authn.Authenticator, image string, enablePrivateRegistries bool, enablePrivateRegistriesTls bool, tlsSecretPath string) (containerregistry.Image, error) { + // if private registries or their appropriate tls configuration are disabled in the config we pull image with default operation otherwise try and use their tls cert's + if !enablePrivateRegistries || strings.HasPrefix(image, defaultRegistry) || !enablePrivateRegistriesTls { + return remote.Image(ref, remote.WithAuth(auth), remote.WithContext(ctx)) + } + tlsFile := "ca.crt" + // Check if mounted secret location contains CA file. + if _, err := os.Stat(tlsSecretPath); os.IsNotExist(err) { + return nil, err + } + if _, errCRT := os.Stat(filepath.Join(tlsSecretPath, "ca.crt")); os.IsNotExist(errCRT) { + if _, errPEM := os.Stat(filepath.Join(tlsSecretPath, "ca.pem")); os.IsNotExist(errPEM) { + return nil, fmt.Errorf("ca.crt not found: %v, and ca.pem also not found: %v", errCRT, errPEM) + } + tlsFile = "ca.pem" + } + // Load the custom TLS configuration + tlsConfig, err := loadTLSConfig(filepath.Join(tlsSecretPath, tlsFile)) + if err != nil { + return nil, err + } + // Create a custom HTTPS transport + transport := createTransport(tlsConfig) + + // Attempt image pull with given custom TLS cert + img, tlsErr := remote.Image(ref, remote.WithAuth(auth), remote.WithContext(ctx), remote.WithTransport(transport)) + if tlsErr != nil { + // Attempt without given custom TLS cert but with default keychain + klog.Errorf("Pulling image %s with the provided TLS Cert has failed with error %v", image, tlsErr) + klog.Infof("Attempting image pull with default keychain instead of provided TLS Cert") + return remote.Image(ref, remote.WithAuth(auth), remote.WithContext(ctx)) + } + return img, tlsErr +} + +func loadTLSConfig(caCertPath string) (*tls.Config, error) { + // Read the CA certificate file + caCert, err := os.ReadFile(caCertPath) + if err != nil { + return nil, err + } + // Append the CA certificate to the system pool + caCertPool := x509.NewCertPool() + if !caCertPool.AppendCertsFromPEM(caCert) { + return nil, fmt.Errorf("failed to append certificates from PEM") + } + // Create a tls.Config with the CA pool + tlsConfig := &tls.Config{ + RootCAs: caCertPool, + } + return tlsConfig, nil +} + +func createTransport(tlsConfig *tls.Config) *http.Transport { + return &http.Transport{ + TLSClientConfig: tlsConfig, + } +} + // retrieveOrCreatePod retrieves or creates a pod for an image. -func (pm *podManager) retrieveOrCreatePod(ctx context.Context, image string, ttl time.Duration, useGenerateName bool, enablePrivateRegistries bool, registryAuthSecretPath string, registryAuthSecretName string) (client.ObjectKey, error) { +func (pm *podManager) retrieveOrCreatePod(ctx context.Context, image string, ttl time.Duration, useGenerateName bool, enablePrivateRegistries bool, registryAuthSecretPath string, registryAuthSecretName string, enablePrivateRegistriesTls bool, tlsSecretPath string) (client.ObjectKey, error) { var de *digestAndEntrypoint var replacePod bool var currentPod *corev1.Pod var err error val, found := pm.imageMetadataCache.Load(image) if !found { - de, err = pm.imageDigestAndEntrypoint(ctx, image, enablePrivateRegistries, registryAuthSecretPath, registryAuthSecretName) + de, err = pm.imageDigestAndEntrypoint(ctx, image, enablePrivateRegistries, registryAuthSecretPath, registryAuthSecretName, enablePrivateRegistriesTls, tlsSecretPath) if err != nil { return client.ObjectKey{}, fmt.Errorf("unable to get the entrypoint for %v: %w", image, err) } diff --git a/func/internal/podevaluator_podmanager_test.go b/func/internal/podevaluator_podmanager_test.go index e51116ca..f85724b2 100644 --- a/func/internal/podevaluator_podmanager_test.go +++ b/func/internal/podevaluator_podmanager_test.go @@ -644,7 +644,7 @@ func TestPodManager(t *testing.T) { fakeServer.evalFunc = tt.evalFunc //Execute the function under test - go pm.getFuncEvalPodClient(ctx, tt.functionImage, time.Hour, tt.useGenerateName, false, "", "auth-secret") + go pm.getFuncEvalPodClient(ctx, tt.functionImage, time.Hour, tt.useGenerateName, false, "/var/tmp/config-secret/.dockerconfigjson", "auth-secret", false, "/var/tmp/tls-secret/") if tt.podPatch != nil { go func() { diff --git a/func/server/server.go b/func/server/server.go index a0a199c0..b722d8ea 100644 --- a/func/server/server.go +++ b/func/server/server.go @@ -38,18 +38,20 @@ const ( ) var ( - port = flag.Int("port", 9445, "The server port") - functions = flag.String("functions", "./functions", "Path to cached functions.") - config = flag.String("config", "./config.yaml", "Path to the config file.") - enablePrivateRegistries = flag.Bool("enable-private-registry", false, "if true enables the use of private registries and their authentication") - registryAuthSecretPath = flag.String("registry-auth-secret-path", "/var/tmp/config-secret/.dockerconfigjson", "The path of the secret used in custom registry authentication") - registryAuthSecretName = flag.String("registry-auth-secret-name", "auth-secret", "The name of the secret used in custom registry authentication") - podCacheConfig = flag.String("pod-cache-config", "/pod-cache-config/pod-cache-config.yaml", "Path to the pod cache config file. The file is map of function name to TTL.") - podNamespace = flag.String("pod-namespace", "porch-fn-system", "Namespace to run KRM functions pods.") - podTTL = flag.Duration("pod-ttl", 30*time.Minute, "TTL for pods before GC.") - scanInterval = flag.Duration("scan-interval", time.Minute, "The interval of GC between scans.") - disableRuntimes = flag.String("disable-runtimes", "", fmt.Sprintf("The runtime(s) to disable. Multiple runtimes should separated by `,`. Available runtimes: `%v`, `%v`.", execRuntime, podRuntime)) - functionPodTemplateName = flag.String("function-pod-template", "", "Configmap that contains a pod specification") + port = flag.Int("port", 9445, "The server port") + functions = flag.String("functions", "./functions", "Path to cached functions.") + config = flag.String("config", "./config.yaml", "Path to the config file.") + enablePrivateRegistries = flag.Bool("enable-private-registries", false, "if true enables the use of private registries and their authentication") + registryAuthSecretPath = flag.String("registry-auth-secret-path", "/var/tmp/config-secret/.dockerconfigjson", "The path of the secret used for authenticating to custom registries") + registryAuthSecretName = flag.String("registry-auth-secret-name", "auth-secret", "The name of the secret used for authenticating to custom registries") + enablePrivateRegistriesTls = flag.Bool("enable-private-registries-tls", false, "if enabled, will prioritize use of user provided TLS secret when accessing registries") + tlsSecretPath = flag.String("tls-secret-path", "/var/tmp/tls-secret/", "The path of the secret used in tls configuration") + podCacheConfig = flag.String("pod-cache-config", "/pod-cache-config/pod-cache-config.yaml", "Path to the pod cache config file. The file is map of function name to TTL.") + podNamespace = flag.String("pod-namespace", "porch-fn-system", "Namespace to run KRM functions pods.") + podTTL = flag.Duration("pod-ttl", 30*time.Minute, "TTL for pods before GC.") + scanInterval = flag.Duration("scan-interval", time.Minute, "The interval of GC between scans.") + disableRuntimes = flag.String("disable-runtimes", "", fmt.Sprintf("The runtime(s) to disable. Multiple runtimes should separated by `,`. Available runtimes: `%v`, `%v`.", execRuntime, podRuntime)) + functionPodTemplateName = flag.String("function-pod-template", "", "Configmap that contains a pod specification") ) func main() { @@ -92,7 +94,7 @@ func run() error { if wrapperServerImage == "" { return fmt.Errorf("environment variable %v must be set to use pod function evaluator runtime", wrapperServerImageEnv) } - podEval, err := internal.NewPodEvaluator(*podNamespace, wrapperServerImage, *scanInterval, *podTTL, *podCacheConfig, *functionPodTemplateName, *enablePrivateRegistries, *registryAuthSecretPath, *registryAuthSecretName) + podEval, err := internal.NewPodEvaluator(*podNamespace, wrapperServerImage, *scanInterval, *podTTL, *podCacheConfig, *functionPodTemplateName, *enablePrivateRegistries, *registryAuthSecretPath, *registryAuthSecretName, *enablePrivateRegistriesTls, *tlsSecretPath) if err != nil { return fmt.Errorf("failed to initialize pod evaluator: %w", err) }