[CDAP-20860] cache provisioned credential in memory in appfabric

cdap-credential-ext-gcp-wi/src/main/java/io/cdap/cdap/security/spi/credential/GcpWorkloadIdentityCredentialProvider.java

@@ -19,6 +19,9 @@
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Stopwatch;
 import com.google.common.base.Strings;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
 import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
 import io.cdap.cdap.api.retry.RetryableException;
@@ -77,6 +80,8 @@ public class GcpWorkloadIdentityCredentialProvider implements CredentialProvider
       LoggerFactory.getLogger(GcpWorkloadIdentityCredentialProvider.class);
   private CredentialProviderContext credentialProviderContext;
   private ApiClient client;
+  private final LoadingCache<ProvisionedCredentialCacheKey,
+      ProvisionedCredential> credentialLoadingCache;
   static final String CONNECT_TIMEOUT_SECS = "k8s.api.client.connect.timeout.secs";
   static final String CONNECT_TIMEOUT_SECS_DEFAULT = "120";
   static final String READ_TIMEOUT_SECS = "k8s.api.client.read.timeout.secs";
@@ -96,6 +101,30 @@ public class GcpWorkloadIdentityCredentialProvider implements CredentialProvider
   private static final String PROVISIONING_FAILURE_ERROR_MESSAGE_FORMAT =
       "Failed to provision credential with identity '%s'";
 
+  /**
+   * Constructs the {@link GcpWorkloadIdentityCredentialProvider}.
+   */
+  public GcpWorkloadIdentityCredentialProvider() {
+    this.credentialLoadingCache = CacheBuilder.newBuilder()
+        // Provisioned credential expire after 60mins, assuming 20% buffer in cache exp (0.8*60).
+        .expireAfterWrite(48, TimeUnit.MINUTES)
+        .build(new CacheLoader<ProvisionedCredentialCacheKey, ProvisionedCredential>() {
+          @Override
+          public ProvisionedCredential load(ProvisionedCredentialCacheKey
+              provisionedCredentialCacheKey) throws Exception {
+            return getProvisionedCredential(provisionedCredentialCacheKey.getNamespaceMeta(),
+                provisionedCredentialCacheKey.getCredentialIdentity(),
+                provisionedCredentialCacheKey.getScopes());
+          }
+        });
+  }
+
+  @VisibleForTesting
+  public LoadingCache<ProvisionedCredentialCacheKey,
+      ProvisionedCredential> getCredentialLoadingCache() {
+    return credentialLoadingCache;
+  }
+
   @Override
   public String getName() {
     return NAME;
@@ -156,23 +185,22 @@ public ProvisionedCredential provision(NamespaceMeta namespaceMeta,
     try {
       while (stopWatch.elapsed(TimeUnit.SECONDS) < timeout) {
         try {
-          return getProvisionedCredential(namespaceMeta, identity, scopes);
-        } catch (RetryableException e) {
+          return getCredentialLoadingCache().get(new ProvisionedCredentialCacheKey(namespaceMeta,
+              identity, scopes));
+        } catch (Exception e) {
+          if (!(e.getCause() instanceof RetryableException)) {
+            throw e;
+          }
           TimeUnit.MILLISECONDS.sleep(delay);
           delay = (long) (delay * (minMultiplier + Math.random() * (maxMultiplier - minMultiplier
               + 1)));
           delay = Math.min(delay, maxDelay);
-        } catch (Exception e) {
-
-          LOG.error(
-              String.format(PROVISIONING_FAILURE_ERROR_MESSAGE_FORMAT, identity.getIdentity()), e);
-
-          throw new CredentialProvisioningException(
-              String.format(PROVISIONING_FAILURE_ERROR_MESSAGE_FORMAT + ": %s",
-                  identity.getIdentity(), e.getMessage()), e);
         }
       }
-    } catch (InterruptedException e) {
+    } catch (Throwable e) {
+      LOG.error(
+          String.format(PROVISIONING_FAILURE_ERROR_MESSAGE_FORMAT, identity.getIdentity()),
+          e);
       throw new CredentialProvisioningException(
           String.format(PROVISIONING_FAILURE_ERROR_MESSAGE_FORMAT + ": %s",
               identity.getIdentity(), e.getMessage()), e);
@@ -186,7 +214,8 @@ public ProvisionedCredential provision(NamespaceMeta namespaceMeta,
         ));
   }
 
-  private ProvisionedCredential getProvisionedCredential(NamespaceMeta namespaceMeta,
+  @VisibleForTesting
+  ProvisionedCredential getProvisionedCredential(NamespaceMeta namespaceMeta,
       CredentialIdentity identity, @Nullable String scopes) throws IOException, ApiException {
 
     // Get k8s namespace from namespace metadata if using a non-default namespace and namespace
@@ -346,7 +375,7 @@ String executeHttpPostRequest(ConnectionProvider connectionProvider, String body
     }
     if (errorResponse) {
       throw new IOException(String.format("Failed to call URL %s with code; response code %d:\n%s",
-          connection.getURL(), connection.getResponseCode(), response.toString()));
+          connection.getURL(), connection.getResponseCode(), response));
     }
     return response.toString();
   }

cdap-credential-ext-gcp-wi/src/main/java/io/cdap/cdap/security/spi/credential/ProvisionedCredentialCacheKey.java

@@ -0,0 +1,81 @@
+/*
+ * Copyright © 2023 Cask Data, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package io.cdap.cdap.security.spi.credential;
+
+import io.cdap.cdap.proto.NamespaceMeta;
+import io.cdap.cdap.proto.credential.CredentialIdentity;
+import java.util.Objects;
+
+/**
+ * Defines the contents of key used for
+ * caching {@link io.cdap.cdap.proto.credential.ProvisionedCredential}.
+ */
+public final class ProvisionedCredentialCacheKey {
+  private final NamespaceMeta namespaceMeta;
+  private final CredentialIdentity credentialIdentity;
+  private final String scopes;
+  private transient Integer hashCode;
+
+  /**
+   * Constructs the {@link ProvisionedCredentialCacheKey}.
+   *
+   * @param namespaceMeta the {@link NamespaceMeta}
+   * @param credentialIdentity the {@link CredentialIdentity}
+   * @param scopes the comma separated list of OAuth scopes.
+   */
+  public ProvisionedCredentialCacheKey(NamespaceMeta namespaceMeta,
+      CredentialIdentity credentialIdentity, String scopes) {
+    this.namespaceMeta = namespaceMeta;
+    this.credentialIdentity = credentialIdentity;
+    this.scopes = scopes;
+  }
+
+  public NamespaceMeta getNamespaceMeta() {
+    return namespaceMeta;
+  }
+
+  public CredentialIdentity getCredentialIdentity() {
+    return credentialIdentity;
+  }
+
+  public String getScopes() {
+    return scopes;
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (!(o instanceof ProvisionedCredentialCacheKey)) {
+      return  false;
+    }
+    ProvisionedCredentialCacheKey that = (ProvisionedCredentialCacheKey) o;
+    return Objects.equals(namespaceMeta.getNamespaceId().getNamespace(),
+        that.namespaceMeta.getNamespaceId().getNamespace())
+        && Objects.equals(credentialIdentity.getSecureValue(),
+        that.getCredentialIdentity().getSecureValue())
+        && Objects.equals(scopes, that.scopes);
+  }
+
+  @Override
+  public int hashCode() {
+    Integer hashCode = this.hashCode;
+    if (hashCode == null) {
+      this.hashCode = hashCode = Objects.hash(namespaceMeta.getNamespaceId().getNamespace(),
+          credentialIdentity.getSecureValue(), scopes);
+    }
+    return hashCode;
+  }
+}

cdap-credential-ext-gcp-wi/src/test/java/io/cdap/cdap/security/spi/credential/GcpWorkloadIdentityCredentialProviderTest.java

@@ -26,6 +26,9 @@
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
 import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
 import io.cdap.cdap.proto.BasicThrowable;
@@ -82,6 +85,21 @@
     GcpWorkloadIdentityCredentialProvider mockedCredentialProvider =
         spy(gcpWorkloadIdentityCredentialProvider);
 
+    LoadingCache<ProvisionedCredentialCacheKey,
+        ProvisionedCredential> cache = CacheBuilder.newBuilder()
+        .build(new CacheLoader<ProvisionedCredentialCacheKey, ProvisionedCredential>() {
+          @Override
+          public ProvisionedCredential load(ProvisionedCredentialCacheKey
+              provisionedCredentialCacheKey) throws Exception {
+            return mockedCredentialProvider.getProvisionedCredential(
+                provisionedCredentialCacheKey.getNamespaceMeta(),
+                provisionedCredentialCacheKey.getCredentialIdentity(),
+                provisionedCredentialCacheKey.getScopes());
+          }
+        });
+
+    doReturn(cache).when(mockedCredentialProvider).getCredentialLoadingCache();
+
     doThrow(new SocketTimeoutException())
         .doThrow(new ConnectException())
         .doReturn(getSecurityTokenServiceResponse())
@@ -178,7 +196,7 @@
   }
 
   @Test(expected = IOException.class)
   public void testExecuteHttpPostRequestHandlesHTTPErrorResponse() throws Exception {
     InputStream errorMessageStream = new ByteArrayInputStream(
         "Some error here".getBytes(StandardCharsets.UTF_8));
     OutputStream outputStream = new ByteArrayOutputStream();