diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/app/guice/AppFabricServiceRuntimeModule.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/app/guice/AppFabricServiceRuntimeModule.java index 4dfdaf0f9873..43918e4b0cf3 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/app/guice/AppFabricServiceRuntimeModule.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/app/guice/AppFabricServiceRuntimeModule.java @@ -116,6 +116,8 @@ import io.cdap.cdap.internal.app.services.RunRecordCorrectorService; import io.cdap.cdap.internal.app.services.RunRecordMonitorService; import io.cdap.cdap.internal.app.services.ScheduledRunRecordCorrectorService; +import io.cdap.cdap.internal.app.sourcecontrol.PullAppsOperationFactory; +import io.cdap.cdap.internal.app.sourcecontrol.PushAppsOperationFactory; import io.cdap.cdap.internal.app.store.DefaultStore; import io.cdap.cdap.internal.bootstrap.guice.BootstrapModules; import io.cdap.cdap.internal.capability.CapabilityModule; @@ -157,6 +159,7 @@ import io.cdap.cdap.security.impersonation.UGIProvider; import io.cdap.cdap.security.impersonation.UnsupportedUGIProvider; import io.cdap.cdap.security.store.SecureStoreHandler; +import io.cdap.cdap.sourcecontrol.ApplicationManager; import io.cdap.cdap.sourcecontrol.guice.SourceControlModule; import io.cdap.cdap.spi.events.StartProgramEvent; import io.cdap.http.HttpHandler; @@ -366,6 +369,12 @@ protected void configure() { .build(Key.get(ConfiguratorFactory.class, Names.named(AppFabric.FACTORY_IMPLEMENTATION_REMOTE))) ); + + bind(ApplicationManager.class).to( + io.cdap.cdap.internal.app.sourcecontrol.LocalApplicationManager.class); + install(new FactoryModuleBuilder().build(PullAppsOperationFactory.class)); + install(new FactoryModuleBuilder().build(PushAppsOperationFactory.class)); + // Used in InMemoryProgramRunDispatcher, TetheringClientHandler install(RemoteAuthenticatorModules.getDefaultModule( TetheringAgentService.REMOTE_TETHERING_AUTHENTICATOR, diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/SourceControlManagementHttpHandler.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/SourceControlManagementHttpHandler.java index 82622b1e7c3e..745b6aa0f4d9 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/SourceControlManagementHttpHandler.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/SourceControlManagementHttpHandler.java @@ -34,7 +34,10 @@ import io.cdap.cdap.proto.ApplicationRecord; import io.cdap.cdap.proto.id.ApplicationReference; import io.cdap.cdap.proto.id.NamespaceId; +import io.cdap.cdap.proto.operation.OperationMeta; +import io.cdap.cdap.proto.sourcecontrol.PullMultipleAppsRequest; import io.cdap.cdap.proto.sourcecontrol.PushAppRequest; +import io.cdap.cdap.proto.sourcecontrol.PushMultipleAppsRequest; import io.cdap.cdap.proto.sourcecontrol.RemoteRepositoryValidationException; import io.cdap.cdap.proto.sourcecontrol.RepositoryConfigRequest; import io.cdap.cdap.proto.sourcecontrol.RepositoryConfigValidationException; @@ -183,6 +186,49 @@ public void pushApp(FullHttpRequest request, HttpResponder responder, } } + /** + * Pushes application configs of requested applications to linked repository in Json format. + * It expects a post body that has a list of application ids and an optional commit message + * E.g. + * + * 
+   * {@code
+   * {
+   *   "appIds": ["app_id_1", "app_id_2"],
+   *   "commitMessage": "pushed application XYZ"
+   * }
+   * }
+   *
+   *
+ * The response will be a {@link OperationMeta} object, which encapsulates the application name, + * version and fileHash. + */ + @POST + @Path("/apps/push") + public void pushApps(FullHttpRequest request, HttpResponder responder, + @PathParam("namespace-id") String namespaceId) throws Exception { + checkSourceControlMultiFeatureFlag(); + NamespaceId namespace = validateNamespaceId(namespaceId); + + PushMultipleAppsRequest appsRequest; + try { + appsRequest = parseBody(request, PushMultipleAppsRequest.class); + } catch (JsonSyntaxException e) { + throw new BadRequestException("Invalid request body.", e); + } + + if (appsRequest == null) { + throw new BadRequestException("Invalid request body."); + } + + if (Strings.isNullOrEmpty(appsRequest.getCommitMessage())) { + throw new BadRequestException("Please specify commit message in the request body."); + } + + OperationMeta operationMeta = sourceControlService.pushApps(namespace, appsRequest); + responder.sendJson(HttpResponseStatus.OK, GSON.toJson(operationMeta)); + } + /** * Pull the requested application from linked repository and deploy in current namespace. */ @@ -202,6 +248,31 @@ public void pullApp(FullHttpRequest request, HttpResponder responder, } } + /** + * Pull the requested applications from linked repository and deploy in current namespace. + */ + @POST + @Path("/apps/pull") + public void pullApps(FullHttpRequest request, HttpResponder responder, + @PathParam("namespace-id") String namespaceId) throws Exception { + checkSourceControlMultiFeatureFlag(); + NamespaceId namespace = validateNamespaceId(namespaceId); + + PullMultipleAppsRequest appsRequest; + try { + appsRequest = parseBody(request, PullMultipleAppsRequest.class); + } catch (JsonSyntaxException e) { + throw new BadRequestException("Invalid request body.", e); + } + + if (appsRequest == null) { + throw new BadRequestException("Invalid request body."); + } + + OperationMeta operationMeta = sourceControlService.pullApps(namespace, appsRequest); + responder.sendJson(HttpResponseStatus.OK, GSON.toJson(operationMeta)); + } + private PushAppRequest validateAndGetAppsRequest(FullHttpRequest request) throws BadRequestException { PushAppRequest appRequest; try { @@ -227,6 +298,13 @@ private void checkSourceControlFeatureFlag() throws ForbiddenException { } } + private void checkSourceControlMultiFeatureFlag() throws ForbiddenException { + checkSourceControlFeatureFlag(); + if (!Feature.SOURCE_CONTROL_MANAGEMENT_MULTIPLE_APPS.isEnabled(featureFlagsProvider)) { + throw new ForbiddenException("Source Control Management for multiple apps feature is not enabled."); + } + } + private NamespaceId validateNamespaceId(String namespaceId) throws BadRequestException { try { return new NamespaceId(namespaceId); diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/SourceControlManagementService.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/SourceControlManagementService.java index 9e24ecac06b2..e26bd047a723 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/SourceControlManagementService.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/SourceControlManagementService.java @@ -16,6 +16,7 @@ package io.cdap.cdap.internal.app.services; +import com.google.common.util.concurrent.ListenableFuture; import com.google.inject.Inject; import io.cdap.cdap.api.artifact.ArtifactSummary; import io.cdap.cdap.api.security.store.SecureStore; @@ -26,6 +27,14 @@ import io.cdap.cdap.common.app.RunIds; import io.cdap.cdap.common.conf.CConfiguration; import io.cdap.cdap.internal.app.deploy.pipeline.ApplicationWithPrograms; +import io.cdap.cdap.internal.app.sourcecontrol.PullAppsOperation; +import io.cdap.cdap.internal.app.sourcecontrol.PullAppsOperationFactory; +import io.cdap.cdap.internal.app.sourcecontrol.PullAppsRequest; +import io.cdap.cdap.internal.app.sourcecontrol.PushAppsOperation; +import io.cdap.cdap.internal.app.sourcecontrol.PushAppsOperationFactory; +import io.cdap.cdap.internal.app.sourcecontrol.PushAppsRequest; +import io.cdap.cdap.internal.operation.OperationException; +import io.cdap.cdap.internal.operation.SynchronousLongRunningOperationContext; import io.cdap.cdap.proto.ApplicationDetail; import io.cdap.cdap.proto.ApplicationRecord; import io.cdap.cdap.proto.artifact.AppRequest; @@ -33,8 +42,13 @@ import io.cdap.cdap.proto.id.ApplicationReference; import io.cdap.cdap.proto.id.KerberosPrincipalId; import io.cdap.cdap.proto.id.NamespaceId; +import io.cdap.cdap.proto.operation.OperationMeta; +import io.cdap.cdap.proto.operation.OperationResource; +import io.cdap.cdap.proto.operation.OperationType; import io.cdap.cdap.proto.security.NamespacePermission; import io.cdap.cdap.proto.security.StandardPermission; +import io.cdap.cdap.proto.sourcecontrol.PullMultipleAppsRequest; +import io.cdap.cdap.proto.sourcecontrol.PushMultipleAppsRequest; import io.cdap.cdap.proto.sourcecontrol.RemoteRepositoryValidationException; import io.cdap.cdap.proto.sourcecontrol.RepositoryConfig; import io.cdap.cdap.proto.sourcecontrol.RepositoryMeta; @@ -62,7 +76,10 @@ import io.cdap.cdap.store.NamespaceTable; import io.cdap.cdap.store.RepositoryTable; import java.io.IOException; +import java.util.HashSet; import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ExecutionException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -79,6 +96,8 @@ public class SourceControlManagementService { private final SourceControlOperationRunner sourceControlOperationRunner; private final ApplicationLifecycleService appLifecycleService; private final Store store; + private final PushAppsOperationFactory pushAppsOperationFactory; + private final PullAppsOperationFactory pullAppsOperationFactory; private static final Logger LOG = LoggerFactory.getLogger(SourceControlManagementService.class); @@ -93,7 +112,8 @@ public SourceControlManagementService(CConfiguration cConf, AuthenticationContext authenticationContext, SourceControlOperationRunner sourceControlOperationRunner, ApplicationLifecycleService applicationLifecycleService, - Store store) { + Store store, PushAppsOperationFactory pushAppsOperationFactory, + PullAppsOperationFactory pullAppsOperationFactory) { this.cConf = cConf; this.secureStore = secureStore; this.transactionRunner = transactionRunner; @@ -102,6 +122,8 @@ public SourceControlManagementService(CConfiguration cConf, this.sourceControlOperationRunner = sourceControlOperationRunner; this.appLifecycleService = applicationLifecycleService; this.store = store; + this.pushAppsOperationFactory = pushAppsOperationFactory; + this.pullAppsOperationFactory = pullAppsOperationFactory; } private RepositoryTable getRepositoryTable(StructuredTableContext context) throws TableNotFoundException { @@ -203,7 +225,7 @@ public PushAppResponse pushApp(ApplicationReference appRef, String commitMessage // TODO: CDAP-20396 RepositoryConfig is currently only accessible from the service layer // Need to fix it and avoid passing it in RepositoryManagerFactory RepositoryConfig repoConfig = getRepositoryMeta(appRef.getParent()).getConfig(); - + // AppLifecycleService already enforces ApplicationDetail Access ApplicationDetail appDetail = appLifecycleService.getLatestAppDetail(appRef, false); @@ -224,7 +246,7 @@ public PushAppResponse pushApp(ApplicationReference appRef, String commitMessage appRef.getApplication(), appRef.getParent(), appLifecycleService.decodeUserId(authenticationContext)); - + SourceControlMeta sourceControlMeta = new SourceControlMeta(pushResponse.getFileHash()); ApplicationId appId = appRef.app(appDetail.getAppVersion()); store.setAppSourceControlMeta(appId, sourceControlMeta); @@ -251,7 +273,7 @@ public ApplicationRecord pullAndDeploy(ApplicationReference appRef) throws Excep accessEnforcer.enforce(appId, authenticationContext.getPrincipal(), StandardPermission.CREATE); accessEnforcer.enforce(appRef.getParent(), authenticationContext.getPrincipal(), NamespacePermission.READ_REPOSITORY); - + PullAppResponse pullResponse = pullAndValidateApplication(appRef); AppRequest appRequest = pullResponse.getAppRequest(); @@ -319,4 +341,73 @@ public RepositoryAppsResponse listApps(NamespaceId namespace) throws NotFoundExc RepositoryConfig repoConfig = getRepositoryMeta(namespace).getConfig(); return sourceControlOperationRunner.list(new NamespaceRepository(namespace, repoConfig)); } + + /** + * The method to push multiple applications in the same namespace to the linked repository. + * + * @param namespace {@link NamespaceId} from where the apps are to be pushed + * @param request {@link PushMultipleAppsRequest} containing the appIds and the commit message + * + * @return {@link OperationMeta} of the operation to push the apps + * @throws NoChangesToPushException when none of the apps have changed since last commit + * @throws NotFoundException when the repository or any of the apps are not found + * @throws InterruptedException when the push operation is inturrupted + * @throws ExecutionException when the push operation execution fails + * @throws OperationException for any exception occuring in the operation logic + */ + public OperationMeta pushApps(NamespaceId namespace, PushMultipleAppsRequest request) + throws NoChangesToPushException, NotFoundException, InterruptedException, + ExecutionException, OperationException { + accessEnforcer.enforce(namespace, authenticationContext.getPrincipal(), + NamespacePermission.WRITE_REPOSITORY); + RepositoryConfig repoConfig = getRepositoryMeta(namespace).getConfig(); + String committer = authenticationContext.getPrincipal().getName(); + PushAppsOperation pushOp = pushAppsOperationFactory.create(new PushAppsRequest( + new HashSet<>(request.getApps()), + repoConfig, + new CommitMeta(committer, committer, System.currentTimeMillis(), request.getCommitMessage()) + )); + + SynchronousLongRunningOperationContext operationContext = new SynchronousLongRunningOperationContext( + namespace.getNamespace(), + OperationType.PUSH_APPS + ); + ListenableFuture> result = pushOp.run(operationContext); + result.get(); + + return operationContext.getOperationMeta(); + } + + /** + * The method to pull multiple applications from the linked repository and deploy them in current namespace. + * + * @param namespace {@link NamespaceId} from where the apps are to be pushed + * @param request {@link PullMultipleAppsRequest} containing the appIds + * + * @return {@link OperationMeta} of the operation to push the apps + * @throws NotFoundException when the repository or any of the apps are not found + * @throws InterruptedException when the push operation is inturrupted + * @throws ExecutionException when the push operation execution fails + * @throws OperationException for any exception occuring in the operation logic + */ + public OperationMeta pullApps(NamespaceId namespace, PullMultipleAppsRequest request) + throws NotFoundException, InterruptedException, + ExecutionException, OperationException { + accessEnforcer.enforce(namespace, authenticationContext.getPrincipal(), + NamespacePermission.READ_REPOSITORY); + RepositoryConfig repoConfig = getRepositoryMeta(namespace).getConfig(); + PullAppsOperation pullOp = pullAppsOperationFactory.create(new PullAppsRequest( + new HashSet<>(request.getApps()), + repoConfig + )); + + SynchronousLongRunningOperationContext operationContext = new SynchronousLongRunningOperationContext( + namespace.getNamespace(), + OperationType.PULL_APPS + ); + ListenableFuture> result = pullOp.run(operationContext); + result.get(); + + return operationContext.getOperationMeta(); + } } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/sourcecontrol/PushAppsOperationFactory.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/sourcecontrol/PushAppsOperationFactory.java index 6a3ce0f3e53d..0f3add565a09 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/sourcecontrol/PushAppsOperationFactory.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/sourcecontrol/PushAppsOperationFactory.java @@ -16,7 +16,6 @@ package io.cdap.cdap.internal.app.sourcecontrol; - /** * Factory interface for creating {@link PushAppsOperation}. * This interface is for Guice assisted binding, hence there will be no concrete implementation of it. @@ -27,7 +26,7 @@ public interface PushAppsOperationFactory { * Returns an implementation of {@link PushAppsOperation} that operates on the given {@link * PushAppsRequest}. * - * @param request contains list of apps to pull + * @param request contains list of apps to push * @return a new instance of {@link PushAppsOperation}. */ PushAppsOperation create(PushAppsRequest request); diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/operation/AbstractLongRunningOperationContext.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/operation/AbstractLongRunningOperationContext.java index de054860b75a..0e55b81e664d 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/operation/AbstractLongRunningOperationContext.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/operation/AbstractLongRunningOperationContext.java @@ -34,4 +34,14 @@ protected AbstractLongRunningOperationContext(OperationRunId runid, OperationTyp this.runId = runid; this.type = operationType; } + + @Override + public OperationRunId getRunId() { + return runId; + } + + @Override + public OperationType getType() { + return type; + } } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/operation/SynchronousLongRunningOperationContext.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/operation/SynchronousLongRunningOperationContext.java new file mode 100644 index 000000000000..3846158efd6d --- /dev/null +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/operation/SynchronousLongRunningOperationContext.java @@ -0,0 +1,44 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.internal.operation; + +import io.cdap.cdap.proto.id.OperationRunId; +import io.cdap.cdap.proto.operation.OperationMeta; +import io.cdap.cdap.proto.operation.OperationResource; +import io.cdap.cdap.proto.operation.OperationType; +import java.time.Instant; +import java.util.HashSet; +import java.util.Set; +import java.util.UUID; + +public class SynchronousLongRunningOperationContext extends AbstractLongRunningOperationContext { + private OperationMeta operationMeta; + + public SynchronousLongRunningOperationContext(String namespace, OperationType operationType) { + super(new OperationRunId(namespace, UUID.randomUUID().toString()), operationType); + this.operationMeta = new OperationMeta(new HashSet<>(), Instant.now(), null); + } + + @Override + public void updateOperationResources(Set resources) { + operationMeta = new OperationMeta(resources, operationMeta.getCreateTime(), Instant.now()); + } + + public OperationMeta getOperationMeta() { + return operationMeta; + } +} diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/http/AppFabricTestBase.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/http/AppFabricTestBase.java index 930339247f94..32aeb5795c9a 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/http/AppFabricTestBase.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/http/AppFabricTestBase.java @@ -109,7 +109,9 @@ import io.cdap.cdap.proto.id.ProgramId; import io.cdap.cdap.proto.id.ProgramReference; import io.cdap.cdap.proto.profile.Profile; +import io.cdap.cdap.proto.sourcecontrol.PullMultipleAppsRequest; import io.cdap.cdap.proto.sourcecontrol.PushAppRequest; +import io.cdap.cdap.proto.sourcecontrol.PushMultipleAppsRequest; import io.cdap.cdap.runtime.spi.profile.ProfileStatus; import io.cdap.cdap.scheduler.CoreSchedulerService; import io.cdap.cdap.scheduler.Scheduler; @@ -1493,12 +1495,26 @@ protected HttpResponse pushApplication(ApplicationReference appRef, String commi appRef.getNamespace(), appRef.getApplication()), GSON.toJson(request)); } + protected HttpResponse pushApplications(String namespace, List apps, String commitMessage) + throws Exception { + PushMultipleAppsRequest request = new PushMultipleAppsRequest(apps, commitMessage); + return doPost(String.format("%s/namespaces/%s/repository/apps/push", + Constants.Gateway.API_VERSION_3, namespace), GSON.toJson(request)); + } + protected HttpResponse pullApplication(ApplicationReference appRef) throws Exception { return doPost(String.format("%s/namespaces/%s/repository/apps/%s/pull", Constants.Gateway.API_VERSION_3, appRef.getNamespace(), appRef.getApplication())); } + protected HttpResponse pullApplications(String namespace, List apps) + throws Exception { + PullMultipleAppsRequest request = new PullMultipleAppsRequest(apps); + return doPost(String.format("%s/namespaces/%s/repository/apps/pull", + Constants.Gateway.API_VERSION_3, namespace), GSON.toJson(request)); + } + protected HttpResponse listApplicationsFromRepository(String namespace) throws Exception { return doGet(String.format("%s/namespaces/%s/repository/apps", Constants.Gateway.API_VERSION_3, namespace)); } diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/http/handlers/SourceControlManagementHttpHandlerTests.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/http/handlers/SourceControlManagementHttpHandlerTests.java index 5dceb237b273..4812dc1151e9 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/http/handlers/SourceControlManagementHttpHandlerTests.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/http/handlers/SourceControlManagementHttpHandlerTests.java @@ -28,14 +28,19 @@ import io.cdap.cdap.common.NotFoundException; import io.cdap.cdap.common.conf.CConfiguration; import io.cdap.cdap.common.id.Id; +import io.cdap.cdap.common.id.Id.Namespace; import io.cdap.cdap.features.Feature; import io.cdap.cdap.gateway.handlers.SourceControlManagementHttpHandler; import io.cdap.cdap.internal.app.services.ApplicationLifecycleService; import io.cdap.cdap.internal.app.services.SourceControlManagementService; import io.cdap.cdap.internal.app.services.http.AppFabricTestBase; +import io.cdap.cdap.internal.app.sourcecontrol.PullAppsOperationFactory; +import io.cdap.cdap.internal.app.sourcecontrol.PushAppsOperationFactory; import io.cdap.cdap.metadata.MetadataSubscriberService; import io.cdap.cdap.proto.ApplicationRecord; import io.cdap.cdap.proto.id.NamespaceId; +import io.cdap.cdap.proto.operation.OperationMeta; +import io.cdap.cdap.proto.operation.OperationResource; import io.cdap.cdap.proto.sourcecontrol.AuthConfig; import io.cdap.cdap.proto.sourcecontrol.AuthType; import io.cdap.cdap.proto.sourcecontrol.PatConfig; @@ -61,8 +66,11 @@ import io.cdap.cdap.sourcecontrol.operationrunner.SourceControlOperationRunner; import io.cdap.cdap.spi.data.transaction.TransactionRunner; import io.cdap.common.http.HttpResponse; +import java.time.Instant; import java.util.Arrays; +import java.util.HashSet; import java.util.Map; +import java.util.stream.Collectors; import javax.annotation.Nullable; import org.junit.Assert; import org.junit.Before; @@ -119,17 +127,20 @@ public SourceControlManagementService provideSourceControlManagementService( AuthenticationContext authenticationContext, SourceControlOperationRunner sourceControlRunner, ApplicationLifecycleService applicationLifecycleService, - Store store) { + Store store, PushAppsOperationFactory pushAppsOpFactory, PullAppsOperationFactory pullAppsOpFactory) { + return Mockito.spy(new SourceControlManagementService(cConf, secureStore, transactionRunner, accessEnforcer, authenticationContext, sourceControlRunner, applicationLifecycleService, - store)); + store, pushAppsOpFactory, + pullAppsOpFactory)); } }); } private static void setScmFeatureFlag(boolean flag) { cConf.setBoolean(FEATURE_FLAG_PREFIX + Feature.SOURCE_CONTROL_MANAGEMENT_GIT.getFeatureFlagString(), flag); + cConf.setBoolean(FEATURE_FLAG_PREFIX + Feature.SOURCE_CONTROL_MANAGEMENT_MULTIPLE_APPS.getFeatureFlagString(), flag); } private void assertResponseCode(int expected, HttpResponse response) { @@ -497,6 +508,149 @@ public void testListAppsNotFound() throws Exception { assertResponseCode(404, response); } + @Test + public void testPushAppsSucceeds() throws Exception { + Id.Application appId1 = Id.Application.from(Id.Namespace.DEFAULT, "appToPush1", "v1"); + Id.Application appId2 = Id.Application.from(Id.Namespace.DEFAULT, "appToPush2", "v1"); + Instant createTime = Instant.now(); + + String commitMessage = "push two apps"; + OperationMeta expectedResponse = new OperationMeta( + Arrays.asList(appId1, appId2).stream() + .map(app -> new OperationResource(app.toEntityId().getEntityName())) + .collect(Collectors.toSet()), createTime, null + ); + + Mockito.doReturn(expectedResponse).when(sourceControlService) + .pushApps(Mockito.any(), Mockito.any()); + HttpResponse response = pushApplications(Namespace.DEFAULT.getId(), + Arrays.asList("appToPush1", "appToPush2"), commitMessage); + + // Assert the app is pushed + assertResponseCode(200, response); + OperationMeta result = readResponse(response, OperationMeta.class); + Assert.assertEquals(result, expectedResponse); + } + + @Test + public void testPushAppsInvalidRequest() throws Exception { + // Push empty commit message + String commitMessage = ""; + HttpResponse response = pushApplications(NamespaceId.DEFAULT.getNamespace(), + Arrays.asList("appToPush1", "appToPush2"), commitMessage); + + // Assert the response + assertResponseCode(400, response); + Assert.assertEquals(response.getResponseBodyAsString(), + "Please specify commit message in the request body."); + } + + @Test + public void testPushAppsNotFound() throws Exception { + // Push two applicatiosn to linked repository + String commitMessage = "push two apps"; + Mockito.doThrow(new NotFoundException("apps not found")).when(sourceControlService) + .pushApps(Mockito.any(), Mockito.any()); + HttpResponse response = pushApplications(NamespaceId.DEFAULT.getNamespace(), + Arrays.asList("appToPush1", "appToPush2"), commitMessage); + + // Assert the app is not found + assertResponseCode(404, response); + Assert.assertEquals(response.getResponseBodyAsString(), "apps not found"); + } + + @Test + public void testPushAppsNoChange() throws Exception { + String commitMessage = "push two apps"; + Mockito.doThrow(new NoChangesToPushException("No changes for apps to push")).when(sourceControlService) + .pushApps(Mockito.any(), Mockito.any()); + HttpResponse response = pushApplications(NamespaceId.DEFAULT.getNamespace(), + Arrays.asList("appToPush1", "appToPush2"), commitMessage); + + assertResponseCode(500, response); + Assert.assertTrue(response.getResponseBodyAsString().contains("No changes for apps to push")); + } + + @Test + public void testPushAppsSourceControlException() throws Exception { + String commitMessage = "push two apps"; + Mockito.doThrow(new SourceControlException("Failed to push apps.")).when(sourceControlService) + .pushApps(Mockito.any(), Mockito.any()); + HttpResponse response = pushApplications(NamespaceId.DEFAULT.getNamespace(), + Arrays.asList("appToPush1", "appToPush2"), commitMessage); + + assertResponseCode(500, response); + Assert.assertTrue(response.getResponseBodyAsString().contains("Failed to push apps.")); + } + + @Test + public void testPushAppsInvalidAuthenticationConfig() throws Exception { + String commitMessage = "push two apps"; + Mockito.doThrow(new AuthenticationConfigException("Repository config not valid")).when(sourceControlService) + .pushApps(Mockito.any(), Mockito.any()); + HttpResponse response = pushApplications(NamespaceId.DEFAULT.getNamespace(), + Arrays.asList("appToPush1", "appToPush2"), commitMessage); + + assertResponseCode(500, response); + Assert.assertTrue(response.getResponseBodyAsString().contains("Repository config not valid")); + } + + @Test + public void testPullAppsSucceeds() throws Exception { + Id.Application appId1 = Id.Application.from(Id.Namespace.DEFAULT, "appToPush1", "v1"); + Id.Application appId2 = Id.Application.from(Id.Namespace.DEFAULT, "appToPush2", "v1"); + + OperationMeta expectedResponse = new OperationMeta( + Arrays.asList(appId1, appId2).stream() + .map(app -> new OperationResource(app.toEntityId().getEntityName())) + .collect(Collectors.toSet()), null, null + ); + + Mockito.doReturn(expectedResponse).when(sourceControlService) + .pullApps(Mockito.any(), Mockito.any()); + HttpResponse response = pullApplications(Namespace.DEFAULT.getId(), + Arrays.asList("appToPush1", "appToPush2")); + + // Assert the app is pulled + assertResponseCode(200, response); + OperationMeta result = readResponse(response, OperationMeta.class); + Assert.assertEquals(result, expectedResponse); + } + + @Test + public void testPullAppsNotFound() throws Exception { + Mockito.doThrow(new NotFoundException("apps not found")).when(sourceControlService) + .pullApps(Mockito.any(), Mockito.any()); + HttpResponse response = pullApplications(NamespaceId.DEFAULT.getNamespace(), + Arrays.asList("appToPush1", "appToPush2")); + + // Assert the app is not found + assertResponseCode(404, response); + Assert.assertEquals(response.getResponseBodyAsString(), "apps not found"); + } + + @Test + public void testPullAppsSourceControlException() throws Exception { + Mockito.doThrow(new SourceControlException("Failed to pull apps.")).when(sourceControlService) + .pullApps(Mockito.any(), Mockito.any()); + HttpResponse response = pullApplications(NamespaceId.DEFAULT.getNamespace(), + Arrays.asList("appToPush1", "appToPush2")); + + assertResponseCode(500, response); + Assert.assertTrue(response.getResponseBodyAsString().contains("Failed to pull apps.")); + } + + @Test + public void testPullAppsInvalidAuthenticationConfig() throws Exception { + Mockito.doThrow(new AuthenticationConfigException("Repository config not valid.")).when(sourceControlService) + .pullApps(Mockito.any(), Mockito.any()); + HttpResponse response = pullApplications(NamespaceId.DEFAULT.getNamespace(), + Arrays.asList("appToPush1", "appToPush2")); + + assertResponseCode(500, response); + Assert.assertTrue(response.getResponseBodyAsString().contains("Repository config not valid.")); + } + private String buildRepoRequestString(Provider provider, String link, String defaultBranch, AuthConfig authConfig, @Nullable String pathPrefix) { Map patJsonMap = ImmutableMap.of( diff --git a/cdap-features/src/main/java/io/cdap/cdap/features/Feature.java b/cdap-features/src/main/java/io/cdap/cdap/features/Feature.java index 4f15fab8a6a7..b637f019a7ed 100644 --- a/cdap-features/src/main/java/io/cdap/cdap/features/Feature.java +++ b/cdap-features/src/main/java/io/cdap/cdap/features/Feature.java @@ -38,6 +38,7 @@ public enum Feature { STREAMING_PIPELINE_NATIVE_STATE_TRACKING("6.8.0", false), PUSHDOWN_TRANSFORMATION_WINDOWAGGREGATION("6.9.1"), SOURCE_CONTROL_MANAGEMENT_GIT("6.9.0"), + SOURCE_CONTROL_MANAGEMENT_MULTIPLE_APPS("6.10.0"), WRANGLER_PRECONDITION_SQL("6.9.1"), WRANGLER_EXECUTION_SQL("6.10.0"), WRANGLER_SCHEMA_MANAGEMENT("6.10.0"), diff --git a/cdap-proto/src/main/java/io/cdap/cdap/proto/sourcecontrol/PullMultipleAppsRequest.java b/cdap-proto/src/main/java/io/cdap/cdap/proto/sourcecontrol/PullMultipleAppsRequest.java new file mode 100644 index 000000000000..db9ec5234ee1 --- /dev/null +++ b/cdap-proto/src/main/java/io/cdap/cdap/proto/sourcecontrol/PullMultipleAppsRequest.java @@ -0,0 +1,35 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.proto.sourcecontrol; + +import io.cdap.cdap.proto.id.NamespaceId; +import java.util.List; + +/** + * The request class to push multiple applications (in the same namespace) to linked git repository. + */ +public class PullMultipleAppsRequest { + private final List apps; + + public PullMultipleAppsRequest(List apps) { + this.apps = apps; + } + + public List getApps() { + return apps; + } +} diff --git a/cdap-proto/src/main/java/io/cdap/cdap/proto/sourcecontrol/PushMultipleAppsRequest.java b/cdap-proto/src/main/java/io/cdap/cdap/proto/sourcecontrol/PushMultipleAppsRequest.java new file mode 100644 index 000000000000..cc55cfeaa0c7 --- /dev/null +++ b/cdap-proto/src/main/java/io/cdap/cdap/proto/sourcecontrol/PushMultipleAppsRequest.java @@ -0,0 +1,41 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.proto.sourcecontrol; + +import io.cdap.cdap.proto.id.NamespaceId; +import java.util.List; + +/** + * The request class to push multiple applications (in the same namespace) to linked git repository. + */ +public class PushMultipleAppsRequest { + private final String commitMessage; + private final List apps; + + public PushMultipleAppsRequest(List apps, String commitMessage) { + this.apps = apps; + this.commitMessage = commitMessage; + } + + public String getCommitMessage() { + return commitMessage; + } + + public List getApps() { + return apps; + } +} diff --git a/cdap-source-control/src/main/java/io/cdap/cdap/sourcecontrol/RepositoryManager.java b/cdap-source-control/src/main/java/io/cdap/cdap/sourcecontrol/RepositoryManager.java index e3fac66bc04b..79225ad915c5 100644 --- a/cdap-source-control/src/main/java/io/cdap/cdap/sourcecontrol/RepositoryManager.java +++ b/cdap-source-control/src/main/java/io/cdap/cdap/sourcecontrol/RepositoryManager.java @@ -261,7 +261,7 @@ public Map commitAndPush(CommitMeta commitMeta, if (fileHashes.size() != filesChanged.size()) { throw new SourceControlException( String.format( - "Failed to get fileHash for %s because some paths are not " + "Failed to get fileHashes for %s because some paths are not " + "found in Git tree", filesChanged)); }