From b66bbb9a829618d98ec1978e3ad4c68ec4ca70c6 Mon Sep 17 00:00:00 2001 From: itsankit-google Date: Wed, 6 Sep 2023 11:01:59 +0000 Subject: [PATCH] create namespaceIdentity and namespaceIdentityHandler --- .../guice/AppFabricServiceRuntimeModule.java | 22 +- .../app/namespace/DefaultNamespaceAdmin.java | 4 + .../app/services/AppFabricServer.java | 54 ++-- .../GcpMetadataHttpHandlerInternal.java | 13 +- .../DefaultCredentialProviderService.java | 11 +- ...ice.java => RemoteCredentialProvider.java} | 35 +-- .../guice/MasterCredentialProviderModule.java | 7 + ...ultNamespaceCredentialProviderService.java | 149 ++++++++++ .../credential/GcpWorkloadIdentityUtil.java | 40 +++ .../NamespaceCredentialProviderService.java | 27 ++ .../RemoteNamespaceCredentialProvider.java | 98 +++++++ .../GcpWorkloadIdentityHttpHandler.java | 266 ++++++++++++++++++ ...cpWorkloadIdentityHttpHandlerInternal.java | 84 ++++++ .../DefaultCredentialProviderServiceTest.java | 50 ++-- .../k8s/KubeMasterEnvironment.java | 4 + .../environment/k8s/AppFabricServiceMain.java | 3 - .../NamespaceCredentialProvider.java | 38 +++ .../credential/NamespaceWorkloadIdentity.java | 47 ++++ .../proto/security/NamespacePermission.java | 12 + 19 files changed, 873 insertions(+), 91 deletions(-) rename cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/{RemoteCredentialProviderService.java => RemoteCredentialProvider.java} (89%) create mode 100644 cdap-app-fabric/src/main/java/io/cdap/cdap/internal/namespace/credential/DefaultNamespaceCredentialProviderService.java create mode 100644 cdap-app-fabric/src/main/java/io/cdap/cdap/internal/namespace/credential/GcpWorkloadIdentityUtil.java create mode 100644 cdap-app-fabric/src/main/java/io/cdap/cdap/internal/namespace/credential/NamespaceCredentialProviderService.java create mode 100644 cdap-app-fabric/src/main/java/io/cdap/cdap/internal/namespace/credential/RemoteNamespaceCredentialProvider.java create mode 100644 cdap-app-fabric/src/main/java/io/cdap/cdap/internal/namespace/credential/handler/GcpWorkloadIdentityHttpHandler.java create mode 100644 cdap-app-fabric/src/main/java/io/cdap/cdap/internal/namespace/credential/handler/GcpWorkloadIdentityHttpHandlerInternal.java create mode 100644 cdap-proto/src/main/java/io/cdap/cdap/proto/credential/NamespaceCredentialProvider.java create mode 100644 cdap-proto/src/main/java/io/cdap/cdap/proto/credential/NamespaceWorkloadIdentity.java diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/app/guice/AppFabricServiceRuntimeModule.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/app/guice/AppFabricServiceRuntimeModule.java index 2e8e524cdb54..4dfdaf0f9873 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/app/guice/AppFabricServiceRuntimeModule.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/app/guice/AppFabricServiceRuntimeModule.java @@ -34,6 +34,7 @@ import com.google.inject.name.Named; import com.google.inject.name.Names; import com.google.inject.util.Modules; +import io.cdap.cdap.api.feature.FeatureFlagsProvider; import io.cdap.cdap.app.deploy.Configurator; import io.cdap.cdap.app.deploy.Manager; import io.cdap.cdap.app.deploy.ManagerFactory; @@ -45,11 +46,13 @@ import io.cdap.cdap.common.conf.CConfiguration; import io.cdap.cdap.common.conf.Constants; import io.cdap.cdap.common.conf.Constants.AppFabric; +import io.cdap.cdap.common.feature.DefaultFeatureFlagsProvider; import io.cdap.cdap.common.guice.RemoteAuthenticatorModules; import io.cdap.cdap.common.runtime.RuntimeModule; import io.cdap.cdap.common.utils.Networks; import io.cdap.cdap.config.guice.ConfigStoreModule; import io.cdap.cdap.data.security.DefaultSecretStore; +import io.cdap.cdap.features.Feature; import io.cdap.cdap.gateway.handlers.AppLifecycleHttpHandler; import io.cdap.cdap.gateway.handlers.AppLifecycleHttpHandlerInternal; import io.cdap.cdap.gateway.handlers.AppStateHandler; @@ -131,6 +134,8 @@ import io.cdap.cdap.internal.events.SparkProgramStatusMetricsProvider; import io.cdap.cdap.internal.events.StartProgramEventReaderExtensionProvider; import io.cdap.cdap.internal.events.StartProgramEventSubscriber; +import io.cdap.cdap.internal.namespace.credential.handler.GcpWorkloadIdentityHttpHandler; +import io.cdap.cdap.internal.namespace.credential.handler.GcpWorkloadIdentityHttpHandlerInternal; import io.cdap.cdap.internal.pipeline.SynchronousPipelineFactory; import io.cdap.cdap.internal.profile.ProfileService; import io.cdap.cdap.internal.provision.ProvisionerModule; @@ -186,7 +191,7 @@ public AppFabricServiceRuntimeModule(CConfiguration cConf) { @Override public Module getInMemoryModules() { - return Modules.combine(new AppFabricServiceModule(), + return Modules.combine(new AppFabricServiceModule(cConf), new CapabilityModule(), new NamespaceAdminModule().getInMemoryModules(), new ConfigStoreModule(), @@ -227,7 +232,7 @@ protected void configure() { @Override public Module getStandaloneModules() { - return Modules.combine(new AppFabricServiceModule(), + return Modules.combine(new AppFabricServiceModule(cConf), new CapabilityModule(), new NamespaceAdminModule().getStandaloneModules(), new ConfigStoreModule(), @@ -281,7 +286,7 @@ protected void configure() { @Override public Module getDistributedModules() { - return Modules.combine(new AppFabricServiceModule(ImpersonationHandler.class), + return Modules.combine(new AppFabricServiceModule(cConf, ImpersonationHandler.class), new CapabilityModule(), new NamespaceAdminModule().getDistributedModules(), new ConfigStoreModule(), @@ -326,9 +331,12 @@ protected void configure() { private static final class AppFabricServiceModule extends AbstractModule { private final List> handlerClasses; + private final CConfiguration cConf; - private AppFabricServiceModule(Class... handlerClasses) { + private AppFabricServiceModule(CConfiguration cConf, + Class... handlerClasses) { this.handlerClasses = ImmutableList.copyOf(handlerClasses); + this.cConf = cConf; } @Override @@ -456,6 +464,12 @@ protected void configure() { handlerBinder.addBinding().to(CredentialProviderHttpHandler.class); handlerBinder.addBinding().to(CredentialProviderHttpHandlerInternal.class); + FeatureFlagsProvider featureFlagsProvider = new DefaultFeatureFlagsProvider(cConf); + if (Feature.NAMESPACED_SERVICE_ACCOUNTS.isEnabled(featureFlagsProvider)) { + handlerBinder.addBinding().to(GcpWorkloadIdentityHttpHandler.class); + handlerBinder.addBinding().to(GcpWorkloadIdentityHttpHandlerInternal.class); + } + for (Class handlerClass : handlerClasses) { handlerBinder.addBinding().to(handlerClass); } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/namespace/DefaultNamespaceAdmin.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/namespace/DefaultNamespaceAdmin.java index 0f8357fbd24b..c079d9001292 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/namespace/DefaultNamespaceAdmin.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/namespace/DefaultNamespaceAdmin.java @@ -415,6 +415,10 @@ public synchronized void updateProperties(NamespaceId namespaceId, NamespaceMeta builder.setDescription(namespaceMeta.getDescription()); } + if (Strings.isNullOrEmpty(existingMeta.getIdentity())) { + builder.setIdentity(getIdentity(namespaceId)); + } + NamespaceConfig config = namespaceMeta.getConfig(); if (config != null && !Strings.isNullOrEmpty(config.getSchedulerQueueName())) { builder.setSchedulerQueueName(config.getSchedulerQueueName()); diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/AppFabricServer.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/AppFabricServer.java index 3659d4d86876..ef489954bcd3 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/AppFabricServer.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/AppFabricServer.java @@ -19,8 +19,10 @@ import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.AbstractIdleService; import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; import com.google.inject.Inject; import com.google.inject.name.Named; +import io.cdap.cdap.api.feature.FeatureFlagsProvider; import io.cdap.cdap.api.metrics.MetricsCollectionService; import io.cdap.cdap.app.runtime.ProgramRuntimeService; import io.cdap.cdap.common.conf.CConfiguration; @@ -28,13 +30,17 @@ import io.cdap.cdap.common.conf.SConfiguration; import io.cdap.cdap.common.discovery.ResolvingDiscoverable; import io.cdap.cdap.common.discovery.URIScheme; +import io.cdap.cdap.common.feature.DefaultFeatureFlagsProvider; import io.cdap.cdap.common.http.CommonNettyHttpServiceFactory; import io.cdap.cdap.common.logging.LoggingContextAccessor; import io.cdap.cdap.common.logging.ServiceLoggingContext; import io.cdap.cdap.common.metrics.MetricsReporterHook; import io.cdap.cdap.common.security.HttpsEnabler; +import io.cdap.cdap.features.Feature; import io.cdap.cdap.internal.app.store.AppMetadataStore; import io.cdap.cdap.internal.bootstrap.BootstrapService; +import io.cdap.cdap.internal.credential.CredentialProviderService; +import io.cdap.cdap.internal.namespace.credential.NamespaceCredentialProviderService; import io.cdap.cdap.internal.provision.ProvisioningService; import io.cdap.cdap.internal.sysapp.SystemAppManagementService; import io.cdap.cdap.proto.id.NamespaceId; @@ -80,6 +86,8 @@ public class AppFabricServer extends AbstractIdleService { private final ProgramRunStatusMonitorService programRunStatusMonitorService; private final RunRecordMonitorService runRecordCounterService; private final CoreSchedulerService coreSchedulerService; + private final CredentialProviderService credentialProviderService; + private final NamespaceCredentialProviderService namespaceCredentialProviderService; private final ProvisioningService provisioningService; private final BootstrapService bootstrapService; private final SystemAppManagementService systemAppManagementService; @@ -113,7 +121,8 @@ public AppFabricServer(CConfiguration cConf, SConfiguration sConf, @Named("appfabric.services.names") Set servicesNames, @Named("appfabric.handler.hooks") Set handlerHookNames, CoreSchedulerService coreSchedulerService, - ProvisioningService provisioningService, + CredentialProviderService credentialProviderService, + NamespaceCredentialProviderService namespaceCredentialProviderService, ProvisioningService provisioningService, BootstrapService bootstrapService, SystemAppManagementService systemAppManagementService, TransactionRunner transactionRunner, @@ -138,6 +147,8 @@ public AppFabricServer(CConfiguration cConf, SConfiguration sConf, this.programRunStatusMonitorService = programRunStatusMonitorService; this.sslEnabled = cConf.getBoolean(Constants.Security.SSL.INTERNAL_ENABLED); this.coreSchedulerService = coreSchedulerService; + this.credentialProviderService = credentialProviderService; + this.namespaceCredentialProviderService = namespaceCredentialProviderService; this.provisioningService = provisioningService; this.bootstrapService = bootstrapService; this.systemAppManagementService = systemAppManagementService; @@ -158,23 +169,28 @@ protected void startUp() throws Exception { new ServiceLoggingContext(NamespaceId.SYSTEM.getNamespace(), Constants.Logging.COMPONENT_NAME, Constants.Service.APP_FABRIC_HTTP)); - Futures.allAsList( - ImmutableList.of( - provisioningService.start(), - applicationLifecycleService.start(), - bootstrapService.start(), - programRuntimeService.start(), - programNotificationSubscriberService.start(), - programStopSubscriberService.start(), - runRecordCorrectorService.start(), - programRunStatusMonitorService.start(), - coreSchedulerService.start(), - runRecordCounterService.start(), - runRecordTimeToLiveService.start(), - sourceControlOperationRunner.start(), - repositoryCleanupService.start() - ) - ).get(); + List> futuresList = new ArrayList<>(); + FeatureFlagsProvider featureFlagsProvider = new DefaultFeatureFlagsProvider(cConf); + if (Feature.NAMESPACED_SERVICE_ACCOUNTS.isEnabled(featureFlagsProvider)) { + futuresList.add(namespaceCredentialProviderService.start()); + } + futuresList.addAll(ImmutableList.of( + provisioningService.start(), + applicationLifecycleService.start(), + bootstrapService.start(), + programRuntimeService.start(), + programNotificationSubscriberService.start(), + programStopSubscriberService.start(), + runRecordCorrectorService.start(), + programRunStatusMonitorService.start(), + coreSchedulerService.start(), + credentialProviderService.start(), + runRecordCounterService.start(), + runRecordTimeToLiveService.start(), + sourceControlOperationRunner.start(), + repositoryCleanupService.start() + )); + Futures.allAsList(futuresList).get(); // Create handler hooks List handlerHooks = handlerHookNames.stream() @@ -231,6 +247,8 @@ protected void shutDown() throws Exception { runRecordTimeToLiveService.stopAndWait(); sourceControlOperationRunner.stopAndWait(); repositoryCleanupService.stopAndWait(); + credentialProviderService.stopAndWait(); + namespaceCredentialProviderService.stopAndWait(); } private Cancellable startHttpService(NettyHttpService httpService) throws Exception { diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/worker/sidecar/GcpMetadataHttpHandlerInternal.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/worker/sidecar/GcpMetadataHttpHandlerInternal.java index d00ec679800e..002019feca5a 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/worker/sidecar/GcpMetadataHttpHandlerInternal.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/worker/sidecar/GcpMetadataHttpHandlerInternal.java @@ -29,11 +29,11 @@ import io.cdap.cdap.common.service.Retries; import io.cdap.cdap.common.service.RetryStrategies; import io.cdap.cdap.gateway.handlers.util.AbstractAppFabricHttpHandler; -import io.cdap.cdap.internal.credential.RemoteCredentialProviderService; +import io.cdap.cdap.internal.namespace.credential.RemoteNamespaceCredentialProvider; import io.cdap.cdap.proto.BasicThrowable; import io.cdap.cdap.proto.codec.BasicThrowableCodec; -import io.cdap.cdap.proto.credential.CredentialProvider; import io.cdap.cdap.proto.credential.CredentialProvisioningException; +import io.cdap.cdap.proto.credential.NamespaceCredentialProvider; import io.cdap.cdap.proto.credential.NotFoundException; import io.cdap.cdap.proto.credential.ProvisionedCredential; import io.cdap.cdap.proto.security.GcpMetadataTaskContext; @@ -64,8 +64,6 @@ @Singleton @Path("/") public class GcpMetadataHttpHandlerInternal extends AbstractAppFabricHttpHandler { - - public static final String GCP_CREDENTIAL_IDENTITY_NAME = "default"; protected static final String METADATA_FLAVOR_HEADER_KEY = "Metadata-Flavor"; protected static final String METADATA_FLAVOR_HEADER_VALUE = "Google"; private static final Logger LOG = LoggerFactory.getLogger(GcpMetadataHttpHandlerInternal.class); @@ -73,7 +71,7 @@ public class GcpMetadataHttpHandlerInternal extends AbstractAppFabricHttpHandler new BasicThrowableCodec()).create(); private final CConfiguration cConf; private final String metadataServiceTokenEndpoint; - private final CredentialProvider credentialProvider; + private final NamespaceCredentialProvider credentialProvider; private final GcpWorkloadIdentityInternalAuthenticator gcpWorkloadIdentityInternalAuthenticator; private GcpMetadataTaskContext gcpMetadataTaskContext; @@ -89,7 +87,7 @@ public GcpMetadataHttpHandlerInternal(CConfiguration cConf, Constants.TaskWorker.METADATA_SERVICE_END_POINT); this.gcpWorkloadIdentityInternalAuthenticator = new GcpWorkloadIdentityInternalAuthenticator(gcpMetadataTaskContext); - this.credentialProvider = new RemoteCredentialProviderService(remoteClientFactory, + this.credentialProvider = new RemoteNamespaceCredentialProvider(remoteClientFactory, this.gcpWorkloadIdentityInternalAuthenticator); } @@ -187,8 +185,7 @@ public void token(HttpRequest request, HttpResponder responder, private GcpTokenResponse fetchTokenFromCredentialProvider(String scopes) throws NotFoundException, IOException, CredentialProvisioningException { ProvisionedCredential provisionedCredential = - this.credentialProvider.provision(gcpMetadataTaskContext.getNamespace(), - GCP_CREDENTIAL_IDENTITY_NAME, scopes); + this.credentialProvider.provision(gcpMetadataTaskContext.getNamespace(), scopes); return new GcpTokenResponse("Bearer", provisionedCredential.get(), Duration.between(Instant.now(), provisionedCredential.getExpiration()).getSeconds()); } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/DefaultCredentialProviderService.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/DefaultCredentialProviderService.java index e85e361871f2..7de096402b7e 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/DefaultCredentialProviderService.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/DefaultCredentialProviderService.java @@ -18,7 +18,7 @@ import com.google.common.util.concurrent.AbstractIdleService; import io.cdap.cdap.common.conf.CConfiguration; -import io.cdap.cdap.common.namespace.NamespaceQueryAdmin; +import io.cdap.cdap.common.namespace.NamespaceAdmin; import io.cdap.cdap.proto.NamespaceMeta; import io.cdap.cdap.proto.credential.CredentialIdentity; import io.cdap.cdap.proto.credential.CredentialProfile; @@ -51,7 +51,7 @@ public class DefaultCredentialProviderService extends AbstractIdleService private final Map credentialProviders; private final CredentialIdentityManager credentialIdentityManager; private final CredentialProfileManager credentialProfileManager; - private final NamespaceQueryAdmin namespaceQueryAdmin; + private final NamespaceAdmin namespaceAdmin; @Inject DefaultCredentialProviderService(CConfiguration cConf, @@ -59,17 +59,18 @@ public class DefaultCredentialProviderService extends AbstractIdleService CredentialProviderLoader credentialProviderLoader, CredentialIdentityManager credentialIdentityManager, CredentialProfileManager credentialProfileManager, - NamespaceQueryAdmin namespaceQueryAdmin) { + NamespaceAdmin namespaceAdmin) { this.cConf = cConf; this.contextAccessEnforcer = contextAccessEnforcer; this.credentialProviders = credentialProviderLoader.loadCredentialProviders(); this.credentialIdentityManager = credentialIdentityManager; this.credentialProfileManager = credentialProfileManager; - this.namespaceQueryAdmin = namespaceQueryAdmin; + this.namespaceAdmin = namespaceAdmin; } @Override protected void startUp() throws Exception { + for (CredentialProvider provider : credentialProviders.values()) { provider.initialize(new DefaultCredentialProviderContext(cConf, provider.getName())); } @@ -100,7 +101,7 @@ public ProvisionedCredential provision(String namespace, String identityName, contextAccessEnforcer.enforce(identityId, StandardPermission.USE); NamespaceMeta namespaceMeta; try { - namespaceMeta = namespaceQueryAdmin.get(new NamespaceId(namespace)); + namespaceMeta = namespaceAdmin.get(new NamespaceId(namespace)); } catch (Exception e) { throw new IOException(String.format("Failed to get namespace '%s' metadata", namespace), e); diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/RemoteCredentialProviderService.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/RemoteCredentialProvider.java similarity index 89% rename from cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/RemoteCredentialProviderService.java rename to cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/RemoteCredentialProvider.java index 6d3457bfbd54..5491452d9d60 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/RemoteCredentialProviderService.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/RemoteCredentialProvider.java @@ -16,7 +16,6 @@ package io.cdap.cdap.internal.credential; -import com.google.common.util.concurrent.AbstractIdleService; import com.google.gson.Gson; import com.google.gson.GsonBuilder; import io.cdap.cdap.api.retry.Idempotency; @@ -28,6 +27,7 @@ import io.cdap.cdap.proto.NamespaceMeta; import io.cdap.cdap.proto.codec.BasicThrowableCodec; import io.cdap.cdap.proto.credential.CredentialIdentity; +import io.cdap.cdap.proto.credential.CredentialProvider; import io.cdap.cdap.proto.credential.CredentialProvisioningException; import io.cdap.cdap.proto.credential.IdentityValidationException; import io.cdap.cdap.proto.credential.NotFoundException; @@ -39,22 +39,21 @@ import joptsimple.internal.Strings; /** - * Remote implementation for {@link CredentialProviderService} used in + * Remote implementation for {@link CredentialProvider} used in * {@link io.cdap.cdap.common.conf.Constants.ArtifactLocalizer}. */ -public class RemoteCredentialProviderService extends AbstractIdleService - implements CredentialProviderService { +public class RemoteCredentialProvider implements CredentialProvider { private static final Gson GSON = new GsonBuilder().registerTypeAdapter(BasicThrowable.class, new BasicThrowableCodec()).create(); private final RemoteClient remoteClient; /** - * Construct the {@link RemoteCredentialProviderService}. + * Construct the {@link RemoteCredentialProvider}. * * @param remoteClientFactory A factory to create {@link RemoteClient}. * @param internalAuthenticator An authenticator to propagate internal identity headers. */ - public RemoteCredentialProviderService(RemoteClientFactory remoteClientFactory, + public RemoteCredentialProvider(RemoteClientFactory remoteClientFactory, InternalAuthenticator internalAuthenticator) { this.remoteClient = remoteClientFactory.createRemoteClient(Constants.Service.APP_FABRIC_HTTP, @@ -62,22 +61,6 @@ public RemoteCredentialProviderService(RemoteClientFactory remoteClientFactory, internalAuthenticator); } - /** - * Start the service. - */ - @Override - protected void startUp() throws Exception { - - } - - /** - * Stop the service. - */ - @Override - protected void shutDown() throws Exception { - - } - /** * Provisions a short-lived credential for the provided identity using the provided identity. * @@ -105,6 +88,14 @@ public ProvisionedCredential provision(String namespace, String identityName, throw new NotFoundException(String.format("Credential Identity %s Not Found.", identityName)); } + + if (response.getResponseCode() != HttpResponseStatus.OK.code()) { + throw new CredentialProvisioningException(String.format( + "Failed to provision credential with response code: %s and error: %s", + response.getResponseCode(), + response.getResponseBodyAsString())); + } + return GSON.fromJson(response.getResponseBodyAsString(), ProvisionedCredential.class); } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/guice/MasterCredentialProviderModule.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/guice/MasterCredentialProviderModule.java index 3ab18f773fd4..7664d834c98f 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/guice/MasterCredentialProviderModule.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/guice/MasterCredentialProviderModule.java @@ -22,7 +22,10 @@ import io.cdap.cdap.internal.credential.CredentialProviderLoader; import io.cdap.cdap.internal.credential.CredentialProviderService; import io.cdap.cdap.internal.credential.DefaultCredentialProviderService; +import io.cdap.cdap.internal.namespace.credential.DefaultNamespaceCredentialProviderService; +import io.cdap.cdap.internal.namespace.credential.NamespaceCredentialProviderService; import io.cdap.cdap.proto.credential.CredentialProvider; +import io.cdap.cdap.proto.credential.NamespaceCredentialProvider; /** * Credential provider module for AppFabric. @@ -35,5 +38,9 @@ protected void configure() { bind(CredentialProviderService.class).to(DefaultCredentialProviderService.class) .in(Scopes.SINGLETON); bind(CredentialProviderLoader.class).to(CredentialProviderExtensionLoader.class); + bind(NamespaceCredentialProvider.class).to(NamespaceCredentialProviderService.class) + .in(Scopes.SINGLETON); + bind(NamespaceCredentialProviderService.class) + .to(DefaultNamespaceCredentialProviderService.class).in(Scopes.SINGLETON); } } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/namespace/credential/DefaultNamespaceCredentialProviderService.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/namespace/credential/DefaultNamespaceCredentialProviderService.java new file mode 100644 index 000000000000..da15d7355a83 --- /dev/null +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/namespace/credential/DefaultNamespaceCredentialProviderService.java @@ -0,0 +1,149 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.internal.namespace.credential; + +import com.google.common.base.Strings; +import com.google.common.util.concurrent.AbstractIdleService; +import com.google.inject.Inject; +import io.cdap.cdap.common.AlreadyExistsException; +import io.cdap.cdap.common.BadRequestException; +import io.cdap.cdap.common.conf.CConfiguration; +import io.cdap.cdap.common.conf.Constants; +import io.cdap.cdap.common.namespace.NamespaceAdmin; +import io.cdap.cdap.internal.credential.CredentialProfileManager; +import io.cdap.cdap.master.environment.MasterEnvironments; +import io.cdap.cdap.master.spi.environment.MasterEnvironment; +import io.cdap.cdap.proto.NamespaceMeta; +import io.cdap.cdap.proto.credential.CredentialProfile; +import io.cdap.cdap.proto.credential.CredentialProvider; +import io.cdap.cdap.proto.credential.CredentialProvisioningException; +import io.cdap.cdap.proto.credential.NotFoundException; +import io.cdap.cdap.proto.credential.ProvisionedCredential; +import io.cdap.cdap.proto.id.CredentialProfileId; +import io.cdap.cdap.proto.id.NamespaceId; +import io.cdap.cdap.proto.security.NamespacePermission; +import io.cdap.cdap.security.spi.authentication.SecurityRequestContext; +import io.cdap.cdap.security.spi.authorization.ContextAccessEnforcer; +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +/** + * Default implementation for {@link NamespaceCredentialProviderService} used in AppFabric. + */ +public class DefaultNamespaceCredentialProviderService extends AbstractIdleService + implements NamespaceCredentialProviderService { + + private static final String CREDENTIAL_PROVIDER_NAME = "gcp-wi-credential-provider"; + private final CredentialProvider credentialProvider; + private final NamespaceAdmin namespaceAdmin; + private final ContextAccessEnforcer contextAccessEnforcer; + private final CredentialProfileManager credentialProfileManager; + private final CConfiguration cConf; + + + @Inject + DefaultNamespaceCredentialProviderService(CConfiguration cConf, + CredentialProvider credentialProvider, ContextAccessEnforcer contextAccessEnforcer, + NamespaceAdmin namespaceAdmin, CredentialProfileManager credentialProfileManager) { + this.cConf = cConf; + this.credentialProvider = credentialProvider; + this.contextAccessEnforcer = contextAccessEnforcer; + this.namespaceAdmin = namespaceAdmin; + this.credentialProfileManager = credentialProfileManager; + } + + /** + * Provisions a short-lived credential for the provided identity using the provided identity. + * + * @param namespace The identity namespace. + * @param scopes A comma separated list of OAuth scopes requested. + * @return A short-lived credential. + * @throws CredentialProvisioningException If provisioning the credential fails. + * @throws IOException If any transport errors occur. + * @throws NotFoundException If the profile or identity are not found. + */ + @Override + public ProvisionedCredential provision(String namespace, String scopes) + throws CredentialProvisioningException, IOException, NotFoundException { + contextAccessEnforcer.enforce(new NamespaceId(namespace), + NamespacePermission.PROVISION_CREDENTIAL); + NamespaceMeta namespaceMeta; + try { + namespaceMeta = namespaceAdmin.get(new NamespaceId(namespace)); + } catch (Exception e) { + throw new IOException(String.format("Failed to get namespace '%s' metadata", + namespace), e); + } + String identityName = + GcpWorkloadIdentityUtil.getWorkloadIdentityName(namespaceMeta.getIdentity()); + switchToInternalUser(); + return credentialProvider.provision(namespace, identityName, scopes); + } + + private void switchToInternalUser() { + SecurityRequestContext.reset(); + } + + /** + * Start the service. + */ + @Override + protected void startUp() throws Exception { + // Updates the namespaces and creates the identities if not exist. + List namespaceMetaList = namespaceAdmin.list(); + for (NamespaceMeta namespaceMeta : namespaceMetaList) { + if (Strings.isNullOrEmpty(namespaceMeta.getIdentity())) { + String identityName = namespaceAdmin.getIdentity(namespaceMeta.getNamespaceId()); + MasterEnvironment masterEnv = MasterEnvironments.getMasterEnvironment(); + if (masterEnv != null + && !cConf.getBoolean(Constants.Namespace.NAMESPACE_CREATION_HOOK_ENABLED)) { + masterEnv.createIdentity(NamespaceId.DEFAULT.getNamespace(), identityName); + } + NamespaceMeta newNamespaceMeta = + new NamespaceMeta.Builder(namespaceMeta) + .setIdentity(identityName) + .build(); + namespaceAdmin.updateProperties(newNamespaceMeta.getNamespaceId(), newNamespaceMeta); + } + } + // create the system profile if not exists. + createSystemProfileIfNotExists(); + } + + private void createSystemProfileIfNotExists() throws BadRequestException, IOException { + CredentialProfileId profileId = new CredentialProfileId(NamespaceId.SYSTEM.getNamespace(), + GcpWorkloadIdentityUtil.SYSTEM_PROFILE_NAME); + CredentialProfile profile = new CredentialProfile( + CREDENTIAL_PROVIDER_NAME, + "System Credential Profile for GCP Workload Identity Credential Provider", + Collections.emptyMap()); + try { + credentialProfileManager.create(profileId, profile); + } catch (AlreadyExistsException e) { + // ignore if the profile already exists. + } + } + + /** + * Stop the service. + */ + @Override + protected void shutDown() throws Exception { + + } +} diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/namespace/credential/GcpWorkloadIdentityUtil.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/namespace/credential/GcpWorkloadIdentityUtil.java new file mode 100644 index 000000000000..8899da769744 --- /dev/null +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/namespace/credential/GcpWorkloadIdentityUtil.java @@ -0,0 +1,40 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.internal.namespace.credential; + +import io.cdap.cdap.proto.credential.NamespaceWorkloadIdentity; + +/** + * Utility class for {@link NamespaceWorkloadIdentity} associated with + * the namespace. + */ +public final class GcpWorkloadIdentityUtil { + + private static final String NAMESPACE_IDENTITY_NAME_PREFIX = "ns-gcp-wi"; + + public static final String SYSTEM_PROFILE_NAME = "ns-gcp-wi"; + + /** + * Returns the namespace workload identity name. + * + * @param identityName The name of identity provided. + * @return namespace workload identity name. + */ + public static String getWorkloadIdentityName(String identityName) { + return String.format("%s-%s", NAMESPACE_IDENTITY_NAME_PREFIX, identityName); + } +} diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/namespace/credential/NamespaceCredentialProviderService.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/namespace/credential/NamespaceCredentialProviderService.java new file mode 100644 index 000000000000..2843b6a6d2e0 --- /dev/null +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/namespace/credential/NamespaceCredentialProviderService.java @@ -0,0 +1,27 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.internal.namespace.credential; + +import com.google.common.util.concurrent.Service; +import io.cdap.cdap.proto.credential.NamespaceCredentialProvider; + +/** + * A service which provides credentials based on identity and profile associated with the namespace. + */ +public interface NamespaceCredentialProviderService extends NamespaceCredentialProvider, Service { + +} diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/namespace/credential/RemoteNamespaceCredentialProvider.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/namespace/credential/RemoteNamespaceCredentialProvider.java new file mode 100644 index 000000000000..f7b03351483c --- /dev/null +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/namespace/credential/RemoteNamespaceCredentialProvider.java @@ -0,0 +1,98 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.internal.namespace.credential; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import io.cdap.cdap.api.retry.Idempotency; +import io.cdap.cdap.common.conf.Constants; +import io.cdap.cdap.common.internal.remote.InternalAuthenticator; +import io.cdap.cdap.common.internal.remote.RemoteClient; +import io.cdap.cdap.common.internal.remote.RemoteClientFactory; +import io.cdap.cdap.internal.credential.RemoteCredentialProvider; +import io.cdap.cdap.proto.BasicThrowable; +import io.cdap.cdap.proto.codec.BasicThrowableCodec; +import io.cdap.cdap.proto.credential.CredentialProvisioningException; +import io.cdap.cdap.proto.credential.NamespaceCredentialProvider; +import io.cdap.cdap.proto.credential.NotFoundException; +import io.cdap.cdap.proto.credential.ProvisionedCredential; +import io.cdap.common.http.HttpMethod; +import io.cdap.common.http.HttpResponse; +import io.netty.handler.codec.http.HttpResponseStatus; +import java.io.IOException; +import joptsimple.internal.Strings; + +/** + * Remote implementation for {@link NamespaceCredentialProvider} used in + * {@link io.cdap.cdap.common.conf.Constants.ArtifactLocalizer}. + */ +public class RemoteNamespaceCredentialProvider implements NamespaceCredentialProvider { + private static final Gson GSON = new GsonBuilder().registerTypeAdapter(BasicThrowable.class, + new BasicThrowableCodec()).create(); + private final RemoteClient remoteClient; + + /** + * Construct the {@link RemoteCredentialProvider}. + * + * @param remoteClientFactory A factory to create {@link RemoteClient}. + * @param internalAuthenticator An authenticator to propagate internal identity headers. + */ + public RemoteNamespaceCredentialProvider(RemoteClientFactory remoteClientFactory, + InternalAuthenticator internalAuthenticator) { + + this.remoteClient = remoteClientFactory.createRemoteClient(Constants.Service.APP_FABRIC_HTTP, + RemoteClientFactory.NO_VERIFY_HTTP_REQUEST_CONFIG, Constants.Gateway.INTERNAL_API_VERSION_3, + internalAuthenticator); + } + + /** + * Provisions a short-lived credential for the provided identity using the provided identity. + * + * @param namespace The identity namespace. + * @param scopes A comma separated list of OAuth scopes requested. + * @return A short-lived credential. + * @throws CredentialProvisioningException If provisioning the credential fails. + * @throws IOException If any transport errors occur. + * @throws NotFoundException If the profile or identity are not found. + */ + @Override + public ProvisionedCredential provision(String namespace, String scopes) + throws CredentialProvisioningException, IOException, NotFoundException { + String url = String.format("namespaces/%s/credentials/workloadIdentity/provision", + namespace); + if (!Strings.isNullOrEmpty(scopes)) { + url = String.format("%s?scopes=%s", url, scopes); + } + io.cdap.common.http.HttpRequest tokenRequest = + remoteClient.requestBuilder(HttpMethod.GET, url).build(); + HttpResponse response = remoteClient.execute(tokenRequest, Idempotency.NONE); + + if (response.getResponseCode() == HttpResponseStatus.NOT_FOUND.code()) { + throw new NotFoundException( + String.format("Credential Identity not found for namespace '%s'.", namespace)); + } + + if (response.getResponseCode() != HttpResponseStatus.OK.code()) { + throw new CredentialProvisioningException(String.format( + "Failed to provision credential with response code: %s and error: %s", + response.getResponseCode(), + response.getResponseBodyAsString())); + } + + return GSON.fromJson(response.getResponseBodyAsString(), ProvisionedCredential.class); + } +} diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/namespace/credential/handler/GcpWorkloadIdentityHttpHandler.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/namespace/credential/handler/GcpWorkloadIdentityHttpHandler.java new file mode 100644 index 000000000000..23f24d760797 --- /dev/null +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/namespace/credential/handler/GcpWorkloadIdentityHttpHandler.java @@ -0,0 +1,266 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.internal.namespace.credential.handler; + +import com.google.common.base.Strings; +import com.google.gson.Gson; +import com.google.gson.JsonSyntaxException; +import com.google.inject.Inject; +import com.google.inject.Singleton; +import io.cdap.cdap.common.AlreadyExistsException; +import io.cdap.cdap.common.BadRequestException; +import io.cdap.cdap.common.NamespaceNotFoundException; +import io.cdap.cdap.common.NotFoundException; +import io.cdap.cdap.common.conf.Constants.Gateway; +import io.cdap.cdap.common.namespace.NamespaceQueryAdmin; +import io.cdap.cdap.internal.credential.CredentialIdentityManager; +import io.cdap.cdap.internal.credential.CredentialProfileManager; +import io.cdap.cdap.internal.namespace.credential.GcpWorkloadIdentityUtil; +import io.cdap.cdap.proto.NamespaceMeta; +import io.cdap.cdap.proto.credential.CredentialIdentity; +import io.cdap.cdap.proto.credential.CredentialProvider; +import io.cdap.cdap.proto.credential.IdentityValidationException; +import io.cdap.cdap.proto.credential.NamespaceWorkloadIdentity; +import io.cdap.cdap.proto.id.CredentialIdentityId; +import io.cdap.cdap.proto.id.NamespaceId; +import io.cdap.cdap.proto.security.NamespacePermission; +import io.cdap.cdap.security.spi.authentication.SecurityRequestContext; +import io.cdap.cdap.security.spi.authorization.ContextAccessEnforcer; +import io.cdap.cdap.security.spi.authorization.UnauthorizedException; +import io.cdap.http.AbstractHttpHandler; +import io.cdap.http.HttpHandler; +import io.cdap.http.HttpResponder; +import io.netty.buffer.ByteBufInputStream; +import io.netty.handler.codec.http.FullHttpRequest; +import io.netty.handler.codec.http.HttpRequest; +import io.netty.handler.codec.http.HttpResponseStatus; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.Reader; +import java.nio.charset.StandardCharsets; +import java.util.Optional; +import javax.ws.rs.DELETE; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.PUT; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; + +/** + * {@link HttpHandler} for namespace identity providers. + */ +@Singleton +@Path(Gateway.API_VERSION_3) +public class GcpWorkloadIdentityHttpHandler extends AbstractHttpHandler { + private static final Gson GSON = new Gson(); + + private final ContextAccessEnforcer accessEnforcer; + private final NamespaceQueryAdmin namespaceQueryAdmin; + private final CredentialIdentityManager credentialIdentityManager; + private final CredentialProfileManager credentialProfileManager; + private final CredentialProvider credentialProvider; + + @Inject + GcpWorkloadIdentityHttpHandler(ContextAccessEnforcer accessEnforcer, + NamespaceQueryAdmin namespaceQueryAdmin, + CredentialIdentityManager credentialIdentityManager, + CredentialProfileManager credentialProfileManager, + CredentialProvider credentialProvider) { + this.accessEnforcer = accessEnforcer; + this.namespaceQueryAdmin = namespaceQueryAdmin; + this.credentialIdentityManager = credentialIdentityManager; + this.credentialProfileManager = credentialProfileManager; + this.credentialProvider = credentialProvider; + } + + /** + * Validates a credential identity. + * + * @param request The HTTP request. + * @param responder The HTTP responder. + * @throws BadRequestException If identity validation fails. + * @throws NotFoundException If the associated profile is not found. + * @throws IOException If transport errors occur. + */ + @POST + @Path("/namespaces/{namespace-id}/credentials/workloadIdentity/validate") + public void validateIdentity(FullHttpRequest request, HttpResponder responder, + @PathParam("namespace-id") String namespace) throws Exception { + accessEnforcer.enforce(new NamespaceId(namespace), NamespacePermission.PROVISION_CREDENTIAL); + NamespaceWorkloadIdentity namespaceWorkloadIdentity = + deserializeRequestContent(request, NamespaceWorkloadIdentity.class); + if (Strings.isNullOrEmpty(namespaceWorkloadIdentity.getIdentity())) { + throw new BadRequestException("Identity cannot be null or empty."); + } + NamespaceMeta namespaceMeta = getNamespaceMeta(namespace); + validateNamespaceIdentity(namespaceMeta, namespaceWorkloadIdentity); + CredentialIdentity credentialIdentity = new CredentialIdentity( + NamespaceId.SYSTEM.getNamespace(), GcpWorkloadIdentityUtil.SYSTEM_PROFILE_NAME, + namespaceWorkloadIdentity.getIdentity(), + namespaceWorkloadIdentity.getServiceAccount()); + switchToInternalUser(); + try { + credentialProvider.validateIdentity(namespaceMeta, credentialIdentity); + } catch (IdentityValidationException e) { + throw new BadRequestException(String.format("Identity validation failed with error: %s", + e.getMessage()), e); + } catch (io.cdap.cdap.proto.credential.NotFoundException e) { + throw new NotFoundException(e.getMessage()); + } + responder.sendJson(HttpResponseStatus.OK, "Namespace identity validated successfully"); + } + + /** + * Fetches a credential identity. + * + * @param request The HTTP request. + * @param responder The HTTP responder. + * @param namespace The identity namespace. + * @throws BadRequestException If the identity name is invalid. + * @throws IOException If transport errors occur. + * @throws NotFoundException If the namespace or identity are not found. + */ + @GET + @Path("/namespaces/{namespace-id}/credentials/workloadIdentity") + public void getIdentity(HttpRequest request, HttpResponder responder, + @PathParam("namespace-id") String namespace) throws Exception { + NamespaceMeta namespaceMeta = getNamespaceMeta(namespace); + CredentialIdentityId credentialIdentityId = createIdentityIdOrPropagate(namespace, + GcpWorkloadIdentityUtil.getWorkloadIdentityName(namespaceMeta.getIdentity())); + switchToInternalUser(); + Optional identity = credentialIdentityManager.get(credentialIdentityId); + if (!identity.isPresent()) { + throw new NotFoundException("Namespace identity not found."); + } + NamespaceWorkloadIdentity workloadIdentity = new NamespaceWorkloadIdentity( + identity.get().getIdentity(), identity.get().getSecureValue()); + responder.sendJson(HttpResponseStatus.OK, GSON.toJson(workloadIdentity)); + } + + /** + * Creates a new identity. + * + * @param request The HTTP request. + * @param responder The HTTP responder. + * @param namespace The identity namespace. + * @throws AlreadyExistsException If the identity exists. + * @throws BadRequestException If the identity name or identity are invalid. + * @throws IOException If transport errors occur. + * @throws NotFoundException If the namespace is not found. + */ + @PUT + @Path("/namespaces/{namespace-id}/credentials/workloadIdentity") + public void createIdentity(FullHttpRequest request, HttpResponder responder, + @PathParam("namespace-id") String namespace) throws Exception { + accessEnforcer.enforce(new NamespaceId(namespace), NamespacePermission.SET_SERVICE_ACCOUNT); + NamespaceWorkloadIdentity namespaceWorkloadIdentity = + deserializeRequestContent(request, NamespaceWorkloadIdentity.class); + if (Strings.isNullOrEmpty(namespaceWorkloadIdentity.getIdentity())) { + throw new BadRequestException("Identity cannot be null or empty."); + } + NamespaceMeta namespaceMeta = getNamespaceMeta(namespace); + validateNamespaceIdentity(namespaceMeta, namespaceWorkloadIdentity); + CredentialIdentityId credentialIdentityId = createIdentityIdOrPropagate(namespace, + GcpWorkloadIdentityUtil.getWorkloadIdentityName(namespaceMeta.getIdentity())); + switchToInternalUser(); + Optional identity = credentialIdentityManager.get(credentialIdentityId); + CredentialIdentity credentialIdentity = new CredentialIdentity( + NamespaceId.SYSTEM.getNamespace(), GcpWorkloadIdentityUtil.SYSTEM_PROFILE_NAME, + namespaceMeta.getIdentity(), namespaceWorkloadIdentity.getServiceAccount()); + if (identity.isPresent()) { + credentialIdentityManager.update(credentialIdentityId, credentialIdentity); + } else { + credentialIdentityManager.create(credentialIdentityId, credentialIdentity); + } + responder.sendStatus(HttpResponseStatus.OK); + } + + /** + * Deletes an identity. + * + * @param request The HTTP request. + * @param responder The HTTP responder. + * @param namespace The identity namespace. + * @throws BadRequestException If the identity name is invalid. + * @throws IOException If transport errors occur. + * @throws NotFoundException If the namespace or identity are not found. + */ + @DELETE + @Path("/namespaces/{namespace-id}/credentials/workloadIdentity") + public void deleteIdentity(HttpRequest request, HttpResponder responder, + @PathParam("namespace-id") String namespace) throws Exception { + accessEnforcer.enforce(new NamespaceId(namespace), NamespacePermission.UNSET_SERVICE_ACCOUNT); + NamespaceMeta namespaceMeta = getNamespaceMeta(namespace); + CredentialIdentityId credentialIdentityId = createIdentityIdOrPropagate(namespace, + GcpWorkloadIdentityUtil.getWorkloadIdentityName(namespaceMeta.getIdentity())); + switchToInternalUser(); + credentialIdentityManager.delete(credentialIdentityId); + responder.sendStatus(HttpResponseStatus.OK); + } + + private NamespaceMeta getNamespaceMeta(String namespace) throws Exception { + if (NamespaceId.SYSTEM.getNamespace().equals(namespace)) { + return NamespaceMeta.SYSTEM; + } + try { + return namespaceQueryAdmin.get(new NamespaceId(namespace)); + } catch (Exception e) { + Throwable cause = e.getCause(); + if (cause instanceof NamespaceNotFoundException || cause instanceof UnauthorizedException) { + throw (Exception) cause; + } + throw new IOException(String.format("Failed to get namespace '%s' metadata", + namespace), e); + } + } + + private void switchToInternalUser() { + SecurityRequestContext.reset(); + } + + private void validateNamespaceIdentity(NamespaceMeta namespaceMeta, NamespaceWorkloadIdentity identity) + throws BadRequestException { + if (!namespaceMeta.getIdentity().equals(identity.getIdentity())) { + throw new BadRequestException("Incorrect value provided for namespace identity."); + } + } + + private CredentialIdentityId createIdentityIdOrPropagate(String namespace, String name) + throws BadRequestException { + try { + return new CredentialIdentityId(namespace, name); + } catch (IllegalArgumentException e) { + throw new BadRequestException(e.getMessage(), e); + } + } + + private T deserializeRequestContent(FullHttpRequest request, Class clazz) + throws BadRequestException { + try (Reader reader = new InputStreamReader(new ByteBufInputStream(request.content()), + StandardCharsets.UTF_8)) { + T content = GSON.fromJson(reader, clazz); + if (content == null) { + throw new BadRequestException("No request object provided; expected class " + + clazz.getName()); + } + return content; + } catch (JsonSyntaxException | IOException e) { + throw new BadRequestException("Unable to parse request: " + e.getMessage(), e); + } + } +} + diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/namespace/credential/handler/GcpWorkloadIdentityHttpHandlerInternal.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/namespace/credential/handler/GcpWorkloadIdentityHttpHandlerInternal.java new file mode 100644 index 000000000000..e2b53f81a0c9 --- /dev/null +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/namespace/credential/handler/GcpWorkloadIdentityHttpHandlerInternal.java @@ -0,0 +1,84 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.internal.namespace.credential.handler; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.inject.Singleton; +import io.cdap.cdap.common.NotFoundException; +import io.cdap.cdap.common.conf.Constants; +import io.cdap.cdap.proto.BasicThrowable; +import io.cdap.cdap.proto.codec.BasicThrowableCodec; +import io.cdap.cdap.proto.credential.CredentialProvisioningException; +import io.cdap.cdap.proto.credential.NamespaceCredentialProvider; +import io.cdap.cdap.security.spi.authorization.ContextAccessEnforcer; +import io.cdap.http.AbstractHttpHandler; +import io.cdap.http.HttpHandler; +import io.cdap.http.HttpResponder; +import io.netty.handler.codec.http.HttpRequest; +import io.netty.handler.codec.http.HttpResponseStatus; +import java.io.IOException; +import javax.inject.Inject; +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.QueryParam; + +/** + * Internal {@link HttpHandler} for credential providers. + */ +@Singleton +@Path(Constants.Gateway.INTERNAL_API_VERSION_3) +public class GcpWorkloadIdentityHttpHandlerInternal extends AbstractHttpHandler { + + private static final Gson GSON = new GsonBuilder().registerTypeAdapter( + BasicThrowable.class, new BasicThrowableCodec()).create(); + + private final NamespaceCredentialProvider credentialProvider; + private final ContextAccessEnforcer accessEnforcer; + + @Inject + GcpWorkloadIdentityHttpHandlerInternal( + ContextAccessEnforcer accessEnforcer, NamespaceCredentialProvider credentialProvider) { + this.credentialProvider = credentialProvider; + this.accessEnforcer = accessEnforcer; + } + + /** + * Provisions a credential for a given identity. + * + * @param request The HTTP request. + * @param responder The HTTP responder. + * @param namespace The namespace of the identity for which to provision a credential. + * @param scopes A comma separated list of OAuth scopes requested. + * @throws CredentialProvisioningException If provisioning fails. + * @throws IOException If transport errors occur. + * @throws NotFoundException If the identity or associated profile are not found. + */ + @GET + @Path("/namespaces/{namespace-id}/credentials/workloadIdentity/provision") + public void provisionCredential(HttpRequest request, HttpResponder responder, + @PathParam("namespace-id") String namespace, @QueryParam("scopes") String scopes) + throws CredentialProvisioningException, IOException, NotFoundException { + try { + responder.sendJson(HttpResponseStatus.OK, + GSON.toJson(credentialProvider.provision(namespace, scopes))); + } catch (io.cdap.cdap.proto.credential.NotFoundException e) { + throw new NotFoundException(e.getMessage()); + } + } +} diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/credential/DefaultCredentialProviderServiceTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/credential/DefaultCredentialProviderServiceTest.java index 6fa64a7dc159..17a3a909d6e5 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/credential/DefaultCredentialProviderServiceTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/credential/DefaultCredentialProviderServiceTest.java @@ -17,7 +17,7 @@ package io.cdap.cdap.internal.credential; import io.cdap.cdap.common.conf.CConfiguration; -import io.cdap.cdap.common.namespace.SimpleNamespaceQueryAdmin; +import io.cdap.cdap.common.namespace.InMemoryNamespaceAdmin; import io.cdap.cdap.proto.NamespaceMeta; import io.cdap.cdap.proto.credential.CredentialIdentity; import io.cdap.cdap.proto.credential.CredentialProvisioningException; @@ -25,8 +25,6 @@ import io.cdap.cdap.proto.credential.NotFoundException; import io.cdap.cdap.proto.id.CredentialIdentityId; import io.cdap.cdap.proto.id.CredentialProfileId; -import java.util.HashMap; -import java.util.Map; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; @@ -37,16 +35,18 @@ public class DefaultCredentialProviderServiceTest extends CredentialProviderTestBase { private static DefaultCredentialProviderService credentialProviderService; + private static final InMemoryNamespaceAdmin namespaceAdmin = new InMemoryNamespaceAdmin(); @BeforeClass public static void startup() { credentialProviderService = new DefaultCredentialProviderService(CConfiguration.create(), contextAccessEnforcer, new MockCredentialProviderLoader(), credentialIdentityManager, - credentialProfileManager, new SimpleNamespaceQueryAdmin(computeNamespaceMap())); + credentialProfileManager, namespaceAdmin); } - private static Map computeNamespaceMap() { - Map namespaceMetaMap = new HashMap<>(); + @Test + public void testProvisionSuccess() throws Exception { + // Create a new profile. String namespace = "testProvisionSuccess"; String identityName = "test"; NamespaceMeta namespaceMeta = new @@ -54,31 +54,7 @@ private static Map computeNamespaceMap() { .setName(namespace) .setIdentity(identityName) .buildWithoutKeytabUriVersion(); - namespaceMetaMap.put(namespace, namespaceMeta); - namespace = "testProvisionWithNotFoundIdentityThrowsException"; - identityName = "does-not-exist"; - namespaceMeta = new - NamespaceMeta.Builder() - .setName(namespace) - .setIdentity(identityName) - .buildWithoutKeytabUriVersion(); - namespaceMetaMap.put(namespace, namespaceMeta); - namespace = "testProvisionFailureThrowsException"; - identityName = "test"; - namespaceMeta = new - NamespaceMeta.Builder() - .setName(namespace) - .setIdentity(identityName) - .buildWithoutKeytabUriVersion(); - namespaceMetaMap.put(namespace, namespaceMeta); - return namespaceMetaMap; - } - - @Test - public void testProvisionSuccess() throws Exception { - // Create a new profile. - String namespace = "testProvisionSuccess"; - String identityName = "test"; + namespaceAdmin.create(namespaceMeta); CredentialProfileId profileId = createDummyProfile(CREDENTIAL_PROVIDER_TYPE_SUCCESS, namespace, "test-profile"); @@ -96,6 +72,12 @@ public void testProvisionSuccess() throws Exception { public void testProvisionWithNotFoundIdentityThrowsException() throws Exception { String namespace = "testProvisionWithNotFoundIdentityThrowsException"; String identityName = "does-not-exist"; + NamespaceMeta namespaceMeta = new + NamespaceMeta.Builder() + .setName(namespace) + .setIdentity(identityName) + .buildWithoutKeytabUriVersion(); + namespaceAdmin.create(namespaceMeta); credentialProviderService.provision(namespace, identityName, null); } @@ -108,6 +90,12 @@ public void testProvisionFailureThrowsException() throws Exception { // Create a new identity. String identityName = "test"; + NamespaceMeta namespaceMeta = new + NamespaceMeta.Builder() + .setName(namespace) + .setIdentity(identityName) + .buildWithoutKeytabUriVersion(); + namespaceAdmin.create(namespaceMeta); CredentialIdentityId id = new CredentialIdentityId(namespace, identityName); CredentialIdentity identity = new CredentialIdentity(profileId.getNamespace(), profileId.getName(), "some-identity", "some-secure-value"); diff --git a/cdap-kubernetes/src/main/java/io/cdap/cdap/master/environment/k8s/KubeMasterEnvironment.java b/cdap-kubernetes/src/main/java/io/cdap/cdap/master/environment/k8s/KubeMasterEnvironment.java index bcd00489f940..45fe5a5eb8c8 100644 --- a/cdap-kubernetes/src/main/java/io/cdap/cdap/master/environment/k8s/KubeMasterEnvironment.java +++ b/cdap-kubernetes/src/main/java/io/cdap/cdap/master/environment/k8s/KubeMasterEnvironment.java @@ -564,6 +564,10 @@ public void createIdentity(String k8sNamespace, String identity) throws ApiExcep coreV1Api.createNamespacedServiceAccount(k8sNamespace, serviceAccount, null, null, null, null); } catch (ApiException e) { + if (e.getCode() == 409) { + // ignore, the SA already exists. + return; + } LOG.error( String.format("Unable to create the service account %s with status %s and body: %s", serviceAccount.getMetadata().getName(), e.getCode(), e.getResponseBody()), e); diff --git a/cdap-master/src/main/java/io/cdap/cdap/master/environment/k8s/AppFabricServiceMain.java b/cdap-master/src/main/java/io/cdap/cdap/master/environment/k8s/AppFabricServiceMain.java index 026acbd48747..ecb6d7eb132d 100644 --- a/cdap-master/src/main/java/io/cdap/cdap/master/environment/k8s/AppFabricServiceMain.java +++ b/cdap-master/src/main/java/io/cdap/cdap/master/environment/k8s/AppFabricServiceMain.java @@ -52,7 +52,6 @@ import io.cdap.cdap.internal.app.services.AppFabricServer; import io.cdap.cdap.internal.app.worker.TaskWorkerServiceLauncher; import io.cdap.cdap.internal.app.worker.system.SystemWorkerServiceLauncher; -import io.cdap.cdap.internal.credential.CredentialProviderService; import io.cdap.cdap.internal.events.EventPublishManager; import io.cdap.cdap.master.spi.environment.MasterEnvironment; import io.cdap.cdap.master.spi.environment.MasterEnvironmentContext; @@ -170,8 +169,6 @@ protected void addServices(Injector injector, List services, // Event publisher could rely on task workers for token generated for security enabled deployments services.add(injector.getInstance(EventPublishManager.class)); - services.add(injector.getInstance(CredentialProviderService.class)); - // Adds the master environment tasks masterEnv.getTasks() .forEach(task -> services.add(new MasterTaskExecutorService(task, masterEnvContext))); diff --git a/cdap-proto/src/main/java/io/cdap/cdap/proto/credential/NamespaceCredentialProvider.java b/cdap-proto/src/main/java/io/cdap/cdap/proto/credential/NamespaceCredentialProvider.java new file mode 100644 index 000000000000..644ed5676df5 --- /dev/null +++ b/cdap-proto/src/main/java/io/cdap/cdap/proto/credential/NamespaceCredentialProvider.java @@ -0,0 +1,38 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.proto.credential; + +import java.io.IOException; + +/** + * Provides a credential based on a profile and identity associated with the namespace. + */ +public interface NamespaceCredentialProvider { + + /** + * Provisions a short-lived credential for the provided identity using the provided identity. + * + * @param namespace The identity namespace. + * @param scopes A comma separated list of OAuth scopes requested. + * @return A short-lived credential. + * @throws CredentialProvisioningException If provisioning the credential fails. + * @throws IOException If any transport errors occur. + * @throws NotFoundException If the profile or identity are not found. + */ + ProvisionedCredential provision(String namespace, String scopes) + throws CredentialProvisioningException, IOException, NotFoundException; +} diff --git a/cdap-proto/src/main/java/io/cdap/cdap/proto/credential/NamespaceWorkloadIdentity.java b/cdap-proto/src/main/java/io/cdap/cdap/proto/credential/NamespaceWorkloadIdentity.java new file mode 100644 index 000000000000..2e8d827a8dd8 --- /dev/null +++ b/cdap-proto/src/main/java/io/cdap/cdap/proto/credential/NamespaceWorkloadIdentity.java @@ -0,0 +1,47 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.proto.credential; + +/** + * Defines an identity for credential provisioning. + */ +public class NamespaceWorkloadIdentity { + + private final String identity; + private final String serviceAccount; + + /** + * Constructs a namespace identity. + * + * @param identity The identity. + * @param serviceAccount The serviceAccount to store for the identity. + */ + public NamespaceWorkloadIdentity(String identity, + String serviceAccount) { + this.identity = identity; + this.serviceAccount = serviceAccount; + } + + public String getIdentity() { + return identity; + } + + public String getServiceAccount() { + return serviceAccount; + } + +} diff --git a/cdap-proto/src/main/java/io/cdap/cdap/proto/security/NamespacePermission.java b/cdap-proto/src/main/java/io/cdap/cdap/proto/security/NamespacePermission.java index 27cbd770f61d..29bf42ded50f 100644 --- a/cdap-proto/src/main/java/io/cdap/cdap/proto/security/NamespacePermission.java +++ b/cdap-proto/src/main/java/io/cdap/cdap/proto/security/NamespacePermission.java @@ -32,6 +32,18 @@ public enum NamespacePermission implements Permission { * Permission to update metadata of the SCM repository of a namespace. */ UPDATE_REPOSITORY_METADATA, + /** + * Permission to set the service account associated with the namespace. + */ + SET_SERVICE_ACCOUNT, + /** + * Permission to unset the service account associated with the namespace. + */ + UNSET_SERVICE_ACCOUNT, + /** + * Permission to provision the credential using the service account associated with the namespace. + */ + PROVISION_CREDENTIAL ; @Override