Merge pull request #15286 from cdapio/CDAP-20750

[CDAP-20750] Implement GcpWorkloadIdentity Credential Provider

cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/worker/sidecar/GcpMetadataHttpHandlerInternal.java

@@ -109,7 +109,11 @@ public void status(HttpRequest request, HttpResponder responder) throws Exceptio
   public void token(HttpRequest request, HttpResponder responder,
       @QueryParam("scopes") String scopes) throws Exception {
 
-    LOG.debug("Token requested for namespace: {}", gcpMetadataTaskContext.getNamespace());
+    if (gcpMetadataTaskContext != null && gcpMetadataTaskContext.getNamespace() != null) {
+      LOG.trace("Token requested for namespace: {}", gcpMetadataTaskContext.getNamespace());
+    } else {
+      LOG.trace("Token requested but namespace not set");
+    }
     // check that metadata header is present in the request.
     if (!request.headers().contains(METADATA_FLAVOR_HEADER_KEY,
         METADATA_FLAVOR_HEADER_VALUE, true)) {

cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/DefaultCredentialProviderContext.java

@@ -18,6 +18,7 @@
 
 import io.cdap.cdap.common.conf.CConfiguration;
 import io.cdap.cdap.common.conf.Constants.CredentialProvider;
+import io.cdap.cdap.common.conf.Constants.Namespace;
 import io.cdap.cdap.security.spi.credential.CredentialProviderContext;
 import java.util.Collections;
 import java.util.Map;
@@ -28,6 +29,7 @@
 public class DefaultCredentialProviderContext implements CredentialProviderContext {
 
   private final Map<String, String> properties;
+  private final boolean isNamespaceCreationHookEnabled;
 
   /**
    * Creates a new context.
@@ -39,6 +41,8 @@ protected DefaultCredentialProviderContext(CConfiguration cConf, String provider
     String prefix = String.format("%s%s.", CredentialProvider.SYSTEM_PROPERTY_PREFIX,
         providerName);
     this.properties = Collections.unmodifiableMap(cConf.getPropsWithPrefix(prefix));
+    this.isNamespaceCreationHookEnabled = cConf.getBoolean(
+        Namespace.NAMESPACE_CREATION_HOOK_ENABLED, false);
   }
 
   /**
@@ -50,4 +54,14 @@ protected DefaultCredentialProviderContext(CConfiguration cConf, String provider
   public Map<String, String> getProperties() {
     return properties;
   }
+
+  /**
+   * Returns a boolean if namespace creation hook is enabled.
+   *
+   * @return true if namespace creation hook is enabled, otherwise false.
+   */
+  @Override
+  public boolean isNamespaceCreationHookEnabled() {
+    return isNamespaceCreationHookEnabled;
+  }
 }

cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/DefaultCredentialProviderService.java

@@ -18,6 +18,7 @@
 
 import com.google.common.util.concurrent.AbstractIdleService;
 import io.cdap.cdap.common.conf.CConfiguration;
+import io.cdap.cdap.proto.NamespaceMeta;
 import io.cdap.cdap.proto.credential.CredentialIdentity;
 import io.cdap.cdap.proto.credential.CredentialProfile;
 import io.cdap.cdap.proto.credential.CredentialProvisioningException;
@@ -33,13 +34,16 @@
 import java.util.Map;
 import java.util.Optional;
 import javax.inject.Inject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Default implementation for {@link CredentialProviderService} used in AppFabric.
  */
 public class DefaultCredentialProviderService extends AbstractIdleService
     implements CredentialProviderService {
 
+  private static final Logger LOG = LoggerFactory.getLogger(DefaultCredentialProviderService.class);
   private final CConfiguration cConf;
   private final ContextAccessEnforcer contextAccessEnforcer;
   private final Map<String, CredentialProvider> credentialProviders;
@@ -64,6 +68,7 @@ protected void startUp() throws Exception {
     for (CredentialProvider provider : credentialProviders.values()) {
       provider.initialize(new DefaultCredentialProviderContext(cConf, provider.getName()));
     }
+    LOG.info("Initialized {} credential providers.", credentialProviders.size());
   }
 
   @Override
@@ -74,63 +79,66 @@ protected void shutDown() throws Exception {
   /**
    * Provisions a credential.
    *
-   * @param namespace    The identity namespace.
+   * @param namespaceMeta    The identity namespace metadata.
    * @param identityName The identity name.
    * @return A provisioned credential.
    * @throws CredentialProvisioningException If provisioning fails in the extension.
    * @throws IOException                     If any transport errors occur.
    * @throws NotFoundException               If the identity or profile are not found.
    */
   @Override
-  public ProvisionedCredential provision(String namespace, String identityName)
+  public ProvisionedCredential provision(NamespaceMeta namespaceMeta, String identityName)
       throws CredentialProvisioningException, IOException, NotFoundException {
-    CredentialIdentityId identityId = new CredentialIdentityId(namespace, identityName);
+    CredentialIdentityId identityId =
+        new CredentialIdentityId(namespaceMeta.getName(), identityName);
     contextAccessEnforcer.enforce(identityId, StandardPermission.USE);
     Optional<CredentialIdentity> optIdentity = credentialIdentityManager.get(identityId);
     if (!optIdentity.isPresent()) {
       throw new NotFoundException(String.format("Credential identity '%s' was not found.",
           identityId.toString()));
     }
     CredentialIdentity identity = optIdentity.get();
-    return validateAndProvisionIdentity(identity);
+    return validateAndProvisionIdentity(namespaceMeta, identity);
   }
 
   /**
    * Validates an identity.
    *
+   * @param namespaceMeta The identity namespace metadata.
    * @param identity The identity to validate.
    * @throws IdentityValidationException If identity validation fails in the extension.
    * @throws IOException                 If any transport errors occur.
    * @throws NotFoundException           If the identity or profile are not found.
    */
   @Override
-  public void validateIdentity(CredentialIdentity identity) throws IdentityValidationException,
-      IOException, NotFoundException {
+  public void validateIdentity(NamespaceMeta namespaceMeta, CredentialIdentity identity)
+      throws IdentityValidationException, IOException, NotFoundException {
     try {
-      validateAndProvisionIdentity(identity);
+      validateAndProvisionIdentity(namespaceMeta, identity);
     } catch (CredentialProvisioningException e) {
       throw new IdentityValidationException(e);
     }
   }
 
-  private ProvisionedCredential validateAndProvisionIdentity(CredentialIdentity identity)
+  private ProvisionedCredential validateAndProvisionIdentity(NamespaceMeta namespaceMeta,
+      CredentialIdentity identity)
       throws CredentialProvisioningException, IOException, NotFoundException {
     CredentialProfileId profileId = new CredentialProfileId(identity.getProfileNamespace(),
         identity.getProfileName());
     contextAccessEnforcer.enforce(profileId, StandardPermission.USE);
     Optional<CredentialProfile> optProfile = credentialProfileManager.get(profileId);
     if (!optProfile.isPresent()) {
-      throw new NotFoundException(String.format("Credential profile '%s' was not found.",
-          profileId.toString()));
+      throw new NotFoundException(
+          String.format("Credential profile '%s' was not found.", profileId));
     }
     CredentialProfile profile = optProfile.get();
     // This is a sanity check which should be impossible to fail.
     String providerType = profile.getCredentialProviderType();
     if (!credentialProviders.containsKey(providerType)) {
-      throw new IllegalStateException(String.format("Unsupported credential provider type "
-          + "'%'", providerType));
+      throw new IllegalStateException(
+          String.format("Unsupported credential provider type '%s'", providerType));
     }
     // Provision and return the credential.
-    return credentialProviders.get(providerType).provision(profile, identity);
+    return credentialProviders.get(providerType).provision(namespaceMeta, profile, identity);
   }
 }

cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/handler/CredentialProviderHttpHandler.java

@@ -16,6 +16,7 @@
 
 package io.cdap.cdap.internal.credential.handler;
 
+import com.google.common.base.Strings;
 import com.google.gson.Gson;
 import com.google.gson.JsonSyntaxException;
 import com.google.inject.Inject;
@@ -29,6 +30,7 @@
 import io.cdap.cdap.common.namespace.NamespaceQueryAdmin;
 import io.cdap.cdap.internal.credential.CredentialIdentityManager;
 import io.cdap.cdap.internal.credential.CredentialProfileManager;
+import io.cdap.cdap.proto.NamespaceMeta;
 import io.cdap.cdap.proto.credential.CreateCredentialIdentityRequest;
 import io.cdap.cdap.proto.credential.CreateCredentialProfileRequest;
 import io.cdap.cdap.proto.credential.CredentialIdentity;
@@ -114,18 +116,36 @@ public void listProviders(HttpRequest request, HttpResponder responder) {
    * @throws IOException         If transport errors occur.
    */
   @POST
-  @Path("/credentials/identities/validate")
-  public void validateIdentity(FullHttpRequest request, HttpResponder responder)
+  @Path("/namespaces/{namespace-id}/credentials/identities/validate")
+  public void validateIdentity(FullHttpRequest request, HttpResponder responder,
+      @PathParam("namespace-id") String namespace)
       throws BadRequestException, NotFoundException, IOException {
     CredentialIdentity identity = deserializeRequestContent(request, CredentialIdentity.class);
+    NamespaceMeta namespaceMeta;
     try {
-      credentialProvider.validateIdentity(identity);
+      namespaceMeta = namespaceQueryAdmin.get(new NamespaceId(namespace));
+    } catch (Exception e) {
+      throw new IOException(String.format("Failed to get namespace '%s' metadata",
+          namespace), e);
+    }
+    if (Strings.isNullOrEmpty(identity.getIdentity())) {
+     throw new BadRequestException("Identity cannot be null or empty.");
+    }
+    if (!identity.getProfileNamespace().equals(namespace)
+        && !identity.getProfileNamespace().equals(NamespaceId.SYSTEM.getNamespace())) {
+      throw new BadRequestException("Cannot validate identity in a namespace that is "
+          + "associated with a profile in a different namespace.");
+    }
+    try {
+      credentialProvider.validateIdentity(namespaceMeta, identity);
     } catch (IdentityValidationException e) {
       throw new BadRequestException(String.format("Identity failed validation with error: %s",
           e.getMessage()), e);
     } catch (io.cdap.cdap.proto.credential.NotFoundException e) {
       throw new NotFoundException(e.getMessage());
     }
+    responder.sendJson(HttpResponseStatus.OK,
+        String.format("Identity '%s' validated successfully", identity.getIdentity()));
   }
 
   /**
@@ -143,7 +163,7 @@ public void listProfiles(HttpRequest request, HttpResponder responder,
       @PathParam("namespace-id") String namespace) throws IOException, NotFoundException {
     accessEnforcer.enforceOnParent(EntityType.CREDENTIAL_PROFILE,
         new NamespaceId(namespace), StandardPermission.LIST);
-    ensureNamespaceExists(namespace);
+    canCreateInNamespace(namespace);
     responder.sendJson(HttpResponseStatus.OK,
         GSON.toJson(credentialProfileManager.list(namespace)));
   }
@@ -166,7 +186,7 @@ public void getProfile(HttpRequest request, HttpResponder responder,
       throws BadRequestException, IOException, NotFoundException {
     final CredentialProfileId profileId = createProfileIdOrPropagate(namespace, profileName);
     accessEnforcer.enforce(profileId, StandardPermission.GET);
-    ensureNamespaceExists(namespace);
+    canCreateInNamespace(namespace);
     Optional<CredentialProfile> profile;
     profile = credentialProfileManager.get(profileId);
     if (profile.isPresent()) {
@@ -200,7 +220,7 @@ public void createProfile(FullHttpRequest request, HttpResponder responder,
       throw new BadRequestException("No profile provided for create request");
     }
     accessEnforcer.enforce(profileId, StandardPermission.CREATE);
-    ensureNamespaceExists(namespace);
+    canCreateInNamespace(namespace);
     credentialProfileManager.create(profileId, createRequest.getProfile());
     responder.sendStatus(HttpResponseStatus.OK);
   }
@@ -223,7 +243,7 @@ public void updateProfile(FullHttpRequest request, HttpResponder responder,
       throws BadRequestException, IOException, NotFoundException {
     final CredentialProfileId profileId = createProfileIdOrPropagate(namespace, profileName);
     accessEnforcer.enforce(profileId, StandardPermission.UPDATE);
-    ensureNamespaceExists(namespace);
+    canCreateInNamespace(namespace);
     final CredentialProfile profile = deserializeRequestContent(request, CredentialProfile.class);
     credentialProfileManager.update(profileId, profile);
     responder.sendStatus(HttpResponseStatus.OK);
@@ -248,7 +268,7 @@ public void deleteProfile(HttpRequest request, HttpResponder responder,
       throws BadRequestException, ConflictException, IOException, NotFoundException {
     final CredentialProfileId profileId = createProfileIdOrPropagate(namespace, profileName);
     accessEnforcer.enforce(profileId, StandardPermission.DELETE);
-    ensureNamespaceExists(namespace);
+    canCreateInNamespace(namespace);
     credentialProfileManager.delete(profileId);
     responder.sendStatus(HttpResponseStatus.OK);
   }
@@ -268,7 +288,7 @@ public void listIdentities(HttpRequest request, HttpResponder responder,
       @PathParam("namespace-id") String namespace) throws IOException, NotFoundException {
     accessEnforcer.enforceOnParent(EntityType.CREDENTIAL_IDENTITY, new NamespaceId(namespace),
         StandardPermission.LIST);
-    ensureNamespaceExists(namespace);
+    canCreateInNamespace(namespace);
     responder.sendJson(HttpResponseStatus.OK,
         GSON.toJson(credentialIdentityManager.list(namespace)));
   }
@@ -293,7 +313,7 @@ public void getIdentity(HttpRequest request, HttpResponder responder,
     final CredentialIdentityId identityId = createIdentityIdOrPropagate(namespace, identityName);
     accessEnforcer.enforce(new CredentialIdentityId(namespace, identityName),
         StandardPermission.GET);
-    ensureNamespaceExists(namespace);
+    canCreateInNamespace(namespace);
     Optional<CredentialIdentity> identity = credentialIdentityManager.get(identityId);
     if (identity.isPresent()) {
       responder.sendJson(HttpResponseStatus.OK, GSON.toJson(identity.get()));
@@ -327,8 +347,13 @@ public void createIdentity(FullHttpRequest request, HttpResponder responder,
       throw new BadRequestException("No identity provided for create request");
     }
     accessEnforcer.enforce(identityId, StandardPermission.CREATE);
-    ensureNamespaceExists(namespace);
+    canCreateInNamespace(namespace);
     validateCredentialIdentity(identity);
+    if (!identity.getProfileNamespace().equals(namespace)
+        && !identity.getProfileNamespace().equals(NamespaceId.SYSTEM.getNamespace())) {
+      throw new BadRequestException("Creation of an identity in a namespace that is "
+          + "associated with a profile in a different namespace is not allowed.");
+    }
     accessEnforcer.enforce(new CredentialProfileId(identity.getProfileNamespace(),
         identity.getProfileName()), StandardPermission.GET);
     credentialIdentityManager.create(identityId, identity);
@@ -355,7 +380,7 @@ public void updateIdentity(FullHttpRequest request, HttpResponder responder,
     final CredentialIdentityId identityId = createIdentityIdOrPropagate(namespace, identityName);
     accessEnforcer.enforce(new CredentialIdentityId(namespace, identityName),
         StandardPermission.UPDATE);
-    ensureNamespaceExists(namespace);
+    canCreateInNamespace(namespace);
     final CredentialIdentity identity = deserializeRequestContent(request,
         CredentialIdentity.class);
     validateCredentialIdentity(identity);
@@ -385,17 +410,21 @@ public void deleteIdentity(HttpRequest request, HttpResponder responder,
     final CredentialIdentityId identityId = createIdentityIdOrPropagate(namespace, identityName);
     accessEnforcer.enforce(new CredentialIdentityId(namespace, identityName),
         StandardPermission.DELETE);
-    ensureNamespaceExists(namespace);
+    canCreateInNamespace(namespace);
     credentialIdentityManager.delete(identityId);
     responder.sendStatus(HttpResponseStatus.OK);
   }
 
-  private void ensureNamespaceExists(String namespace)
+  private void canCreateInNamespace(String namespace)
       throws IOException, NamespaceNotFoundException {
     NamespaceId namespaceId;
     Boolean namespaceFound;
     try {
       namespaceId = new NamespaceId(namespace);
+      if (NamespaceId.SYSTEM.equals(namespaceId)) {
+        // return true for SYSTEM namespace.
+        return;
+      }
       namespaceFound = namespaceQueryAdmin.exists(namespaceId);
     } catch (Exception e) {
       throw new IOException(String.format("Failed to check if namespace '%s' exists",

cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/handler/CredentialProviderHttpHandlerInternal.java

@@ -21,8 +21,11 @@
 import com.google.inject.Singleton;
 import io.cdap.cdap.common.NotFoundException;
 import io.cdap.cdap.common.conf.Constants;
+import io.cdap.cdap.common.namespace.NamespaceQueryAdmin;
+import io.cdap.cdap.proto.NamespaceMeta;
 import io.cdap.cdap.proto.credential.CredentialProvider;
 import io.cdap.cdap.proto.credential.CredentialProvisioningException;
+import io.cdap.cdap.proto.id.NamespaceId;
 import io.cdap.http.AbstractHttpHandler;
 import io.cdap.http.HttpHandler;
 import io.cdap.http.HttpResponder;
@@ -46,10 +49,13 @@ public class CredentialProviderHttpHandlerInternal extends AbstractHttpHandler {
       new InstantEpochSecondsTypeAdapter()).create();
 
   private final CredentialProvider credentialProvider;
+  private final NamespaceQueryAdmin namespaceQueryAdmin;
 
   @Inject
-  CredentialProviderHttpHandlerInternal(CredentialProvider credentialProvider) {
+  CredentialProviderHttpHandlerInternal(CredentialProvider credentialProvider,
+      NamespaceQueryAdmin namespaceQueryAdmin) {
     this.credentialProvider = credentialProvider;
+    this.namespaceQueryAdmin = namespaceQueryAdmin;
   }
 
   /**
@@ -68,9 +74,16 @@ public class CredentialProviderHttpHandlerInternal extends AbstractHttpHandler {
   public void provisionCredential(HttpRequest request, HttpResponder responder,
       @PathParam("namespace-id") String namespace, @PathParam("identity-name") String identityName)
       throws CredentialProvisioningException, IOException, NotFoundException {
+    NamespaceMeta namespaceMeta;
+    try {
+      namespaceMeta = namespaceQueryAdmin.get(new NamespaceId(namespace));
+    } catch (Exception e) {
+     throw new IOException(String.format("Failed to get namespace '%s' metadata",
+         namespace), e);
+    }
     try {
       responder.sendJson(HttpResponseStatus.OK,
-          GSON.toJson(credentialProvider.provision(namespace, identityName)));
+          GSON.toJson(credentialProvider.provision(namespaceMeta, identityName)));
     } catch (io.cdap.cdap.proto.credential.NotFoundException e) {
       throw new NotFoundException(e.getMessage());
     }

cdap-app-fabric/src/test/java/io/cdap/cdap/internal/credential/CredentialProviderTestBase.java

@@ -101,14 +101,14 @@ protected void configure() {
         .getInstance(StructuredTableAdmin.class));
     // Setup mock credential providers.
     CredentialProvider mockCredentialProvider = mock(CredentialProvider.class);
-    when(mockCredentialProvider.provision(any(), any())).thenReturn(RETURNED_TOKEN);
+    when(mockCredentialProvider.provision(any(), any(), any())).thenReturn(RETURNED_TOKEN);
     CredentialProvider validationFailureMockCredentialProvider = mock(CredentialProvider.class);
-    when(validationFailureMockCredentialProvider.provision(any(), any()))
+    when(validationFailureMockCredentialProvider.provision(any(), any(), any()))
         .thenReturn(RETURNED_TOKEN);
     doThrow(new ProfileValidationException("profile validation always fails with this provider"))
         .when(validationFailureMockCredentialProvider).validateProfile(any());
     CredentialProvider provisionFailureMockCredentialProvider = mock(CredentialProvider.class);
-    when(provisionFailureMockCredentialProvider.provision(any(), any()))
+    when(provisionFailureMockCredentialProvider.provision(any(), any(), any()))
         .thenThrow(new CredentialProvisioningException("provisioning always fails with this "
             + "provider"));
     credentialProviders.put(CREDENTIAL_PROVIDER_TYPE_SUCCESS, mockCredentialProvider);