diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/app/store/Store.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/app/store/Store.java index 6f88c48b6685..b99639e2b289 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/app/store/Store.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/app/store/Store.java @@ -368,6 +368,16 @@ List getRuns(Collection programs, ProgramRunSt */ int addApplication(ApplicationId id, ApplicationMeta meta) throws ConflictException; + /** + * Marks existing applications as latest. + * + * @param applicationIds List of application ids + * @throws IOException if the apps cannot be marked latest because of any IO failure + * @throws ApplicationNotFoundException when any of the applications is not found + */ + void markApplicationsLatest(Collection applicationIds) + throws IOException, ApplicationNotFoundException; + /** * Return a list of program specifications that are deleted comparing the specification in the store with the * spec that is passed. diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/AbstractAppLifecycleHttpHandler.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/AbstractAppLifecycleHttpHandler.java new file mode 100644 index 000000000000..ab3dd3476ef4 --- /dev/null +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/AbstractAppLifecycleHttpHandler.java @@ -0,0 +1,215 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.gateway.handlers; + +import com.google.common.base.Throwables; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import io.cdap.cdap.api.artifact.ArtifactSummary; +import io.cdap.cdap.api.dataset.DatasetManagementException; +import io.cdap.cdap.api.security.AccessException; +import io.cdap.cdap.app.runtime.ProgramController; +import io.cdap.cdap.app.runtime.ProgramRuntimeService; +import io.cdap.cdap.common.ArtifactNotFoundException; +import io.cdap.cdap.common.BadRequestException; +import io.cdap.cdap.common.ConflictException; +import io.cdap.cdap.common.InvalidArtifactException; +import io.cdap.cdap.common.NamespaceNotFoundException; +import io.cdap.cdap.common.conf.CConfiguration; +import io.cdap.cdap.common.conf.Constants; +import io.cdap.cdap.common.http.AbstractBodyConsumer; +import io.cdap.cdap.common.io.CaseInsensitiveEnumTypeAdapterFactory; +import io.cdap.cdap.common.namespace.NamespaceQueryAdmin; +import io.cdap.cdap.gateway.handlers.util.AbstractAppFabricHttpHandler; +import io.cdap.cdap.internal.app.deploy.ProgramTerminator; +import io.cdap.cdap.internal.app.deploy.pipeline.ApplicationWithPrograms; +import io.cdap.cdap.internal.app.services.ApplicationLifecycleService; +import io.cdap.cdap.proto.ApplicationRecord; +import io.cdap.cdap.proto.artifact.AppRequest; +import io.cdap.cdap.proto.id.ApplicationId; +import io.cdap.cdap.proto.id.EntityId; +import io.cdap.cdap.proto.id.KerberosPrincipalId; +import io.cdap.cdap.proto.id.NamespaceId; +import io.cdap.cdap.proto.id.ProgramId; +import io.cdap.cdap.security.spi.authorization.UnauthorizedException; +import io.cdap.http.BodyConsumer; +import io.cdap.http.HttpResponder; +import io.netty.handler.codec.http.HttpResponseStatus; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.util.Optional; +import javax.annotation.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * The abstract base class for the {@link AppLifecycleHttpHandler} and {@link AppLifecycleHttpHandlerInternal} + * It contains the common methods used in both handlers mentioned above. + */ +public abstract class AbstractAppLifecycleHttpHandler extends AbstractAppFabricHttpHandler { + protected static final Gson GSON = new Gson(); + protected static final Gson DECODE_GSON = new GsonBuilder() + .registerTypeAdapterFactory(new CaseInsensitiveEnumTypeAdapterFactory()) + .create(); + protected static final Logger LOG = LoggerFactory.getLogger(AbstractAppLifecycleHttpHandler.class); + + protected final CConfiguration configuration; + protected final NamespaceQueryAdmin namespaceQueryAdmin; + protected final ProgramRuntimeService runtimeService; + protected final ApplicationLifecycleService applicationLifecycleService; + protected final File tmpDir; + + /** + * Constructor for the abstract class. + * + * @param configuration CConfiguration, passed from the derived class where it's injected + * @param namespaceQueryAdmin passed from the derived class where it's injected + * @param runtimeService passed from the derived class where it's injected + */ + public AbstractAppLifecycleHttpHandler(CConfiguration configuration, + NamespaceQueryAdmin namespaceQueryAdmin, + ProgramRuntimeService runtimeService, + ApplicationLifecycleService applicationLifecycleService) { + this.configuration = configuration; + this.namespaceQueryAdmin = namespaceQueryAdmin; + this.runtimeService = runtimeService; + this.applicationLifecycleService = applicationLifecycleService; + this.tmpDir = new File(new File(configuration.get(Constants.CFG_LOCAL_DATA_DIR)), + configuration.get(Constants.AppFabric.TEMP_DIR)).getAbsoluteFile(); + } + + protected ApplicationId validateApplicationVersionId(NamespaceId namespaceId, String appId, + String versionId) throws BadRequestException { + if (appId == null) { + throw new BadRequestException("Path parameter app-id cannot be empty"); + } + if (!EntityId.isValidId(appId)) { + throw new BadRequestException(String.format("Invalid app name '%s'", appId)); + } + if (versionId == null) { + throw new BadRequestException("Path parameter version-id cannot be empty"); + } + if (EntityId.isValidVersionId(versionId)) { + return namespaceId.app(appId, versionId); + } + throw new BadRequestException(String.format("Invalid version '%s'", versionId)); + } + + protected NamespaceId validateNamespace(@Nullable String namespace) + throws BadRequestException, NamespaceNotFoundException, AccessException { + + if (namespace == null) { + throw new BadRequestException("Path parameter namespace-id cannot be empty"); + } + + NamespaceId namespaceId; + try { + namespaceId = new NamespaceId(namespace); + } catch (IllegalArgumentException e) { + throw new BadRequestException(String.format("Invalid namespace '%s'", namespace), e); + } + + try { + if (!namespaceId.equals(NamespaceId.SYSTEM)) { + namespaceQueryAdmin.get(namespaceId); + } + } catch (NamespaceNotFoundException | AccessException e) { + throw e; + } catch (Exception e) { + // This can only happen when NamespaceAdmin uses HTTP calls to interact with namespaces. + // In AppFabricServer, NamespaceAdmin is bound to DefaultNamespaceAdmin, which interacts directly with the MDS. + // Hence, this exception will never be thrown + throw Throwables.propagate(e); + } + return namespaceId; + } + + protected ProgramTerminator createProgramTerminator() { + return programId -> { + switch (programId.getType()) { + case SERVICE: + case WORKER: + killProgramIfRunning(programId); + break; + default: + break; + } + }; + } + + protected void killProgramIfRunning(ProgramId programId) { + ProgramRuntimeService.RuntimeInfo programRunInfo = findRuntimeInfo(programId, runtimeService); + if (programRunInfo != null) { + ProgramController controller = programRunInfo.getController(); + controller.kill(); + } + } + + protected ApplicationRecord getApplicationRecord(ApplicationWithPrograms deployedApp) { + return new ApplicationRecord( + ArtifactSummary.from(deployedApp.getArtifactId().toApiArtifactId()), + deployedApp.getApplicationId().getApplication(), + deployedApp.getApplicationId().getVersion(), + deployedApp.getSpecification().getDescription(), + Optional.ofNullable(deployedApp.getOwnerPrincipal()).map(KerberosPrincipalId::getPrincipal) + .orElse(null), + deployedApp.getChangeDetail(), null); + } + + protected BodyConsumer deployAppFromArtifact( + final ApplicationId appId, + final boolean skipMarkingLatest) + throws IOException { + return new AbstractBodyConsumer( + File.createTempFile("apprequest-" + appId, ".json", tmpDir)) { + @Override + protected void onFinish(HttpResponder responder, File uploadedFile) { + try (FileReader fileReader = new FileReader(uploadedFile)) { + AppRequest appRequest = DECODE_GSON.fromJson(fileReader, AppRequest.class); + + try { + ApplicationWithPrograms app = applicationLifecycleService.deployApp(appId, appRequest, + null, createProgramTerminator(), skipMarkingLatest); + responder.sendJson(HttpResponseStatus.OK, GSON.toJson(getApplicationRecord(app))); + } catch (DatasetManagementException e) { + if (e.getCause() instanceof UnauthorizedException) { + throw (UnauthorizedException) e.getCause(); + } else { + throw e; + } + } + } catch (ArtifactNotFoundException e) { + responder.sendString(HttpResponseStatus.NOT_FOUND, e.getMessage()); + } catch (ConflictException e) { + responder.sendString(HttpResponseStatus.CONFLICT, e.getMessage()); + } catch (UnauthorizedException e) { + responder.sendString(HttpResponseStatus.FORBIDDEN, e.getMessage()); + } catch (InvalidArtifactException e) { + responder.sendString(HttpResponseStatus.BAD_REQUEST, e.getMessage()); + } catch (IOException e) { + LOG.error("Error reading request body for creating app {}.", appId, e); + responder.sendString(HttpResponseStatus.INTERNAL_SERVER_ERROR, String.format( + "Error while reading json request body for app %s.", appId)); + } catch (Exception e) { + LOG.error("Deploy failure", e); + responder.sendString(HttpResponseStatus.BAD_REQUEST, e.getMessage()); + } + } + }; + } +} diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/AppLifecycleHttpHandler.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/AppLifecycleHttpHandler.java index 6f10637900d3..4638030dba7b 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/AppLifecycleHttpHandler.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/AppLifecycleHttpHandler.java @@ -19,11 +19,8 @@ import com.google.common.base.Splitter; import com.google.common.base.Strings; -import com.google.common.base.Throwables; import com.google.common.collect.Iterables; import com.google.common.reflect.TypeToken; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonObject; @@ -32,17 +29,13 @@ import com.google.inject.Inject; import com.google.inject.Singleton; import io.cdap.cdap.api.artifact.ArtifactScope; -import io.cdap.cdap.api.artifact.ArtifactSummary; -import io.cdap.cdap.api.dataset.DatasetManagementException; import io.cdap.cdap.api.feature.FeatureFlagsProvider; import io.cdap.cdap.api.security.AccessException; -import io.cdap.cdap.app.runtime.ProgramController; import io.cdap.cdap.app.runtime.ProgramRuntimeService; import io.cdap.cdap.app.store.ApplicationFilter; import io.cdap.cdap.app.store.ScanApplicationsRequest; import io.cdap.cdap.common.ApplicationNotFoundException; import io.cdap.cdap.common.ArtifactAlreadyExistsException; -import io.cdap.cdap.common.ArtifactNotFoundException; import io.cdap.cdap.common.BadRequestException; import io.cdap.cdap.common.ConflictException; import io.cdap.cdap.common.InvalidArtifactException; @@ -56,15 +49,12 @@ import io.cdap.cdap.common.feature.DefaultFeatureFlagsProvider; import io.cdap.cdap.common.http.AbstractBodyConsumer; import io.cdap.cdap.common.id.Id; -import io.cdap.cdap.common.io.CaseInsensitiveEnumTypeAdapterFactory; import io.cdap.cdap.common.namespace.NamespacePathLocator; import io.cdap.cdap.common.namespace.NamespaceQueryAdmin; import io.cdap.cdap.common.security.AuditDetail; import io.cdap.cdap.common.security.AuditPolicy; import io.cdap.cdap.common.utils.DirUtils; import io.cdap.cdap.features.Feature; -import io.cdap.cdap.gateway.handlers.util.AbstractAppFabricHttpHandler; -import io.cdap.cdap.internal.app.deploy.ProgramTerminator; import io.cdap.cdap.internal.app.deploy.pipeline.ApplicationWithPrograms; import io.cdap.cdap.internal.app.runtime.artifact.WriteConflictException; import io.cdap.cdap.internal.app.services.ApplicationLifecycleService; @@ -78,7 +68,6 @@ import io.cdap.cdap.proto.id.EntityId; import io.cdap.cdap.proto.id.KerberosPrincipalId; import io.cdap.cdap.proto.id.NamespaceId; -import io.cdap.cdap.proto.id.ProgramId; import io.cdap.cdap.proto.security.StandardPermission; import io.cdap.cdap.security.spi.authentication.AuthenticationContext; import io.cdap.cdap.security.spi.authorization.AccessEnforcer; @@ -97,7 +86,6 @@ import io.netty.handler.codec.http.HttpResponseStatus; import java.io.ByteArrayOutputStream; import java.io.File; -import java.io.FileReader; import java.io.IOException; import java.io.InputStreamReader; import java.io.OutputStreamWriter; @@ -125,23 +113,13 @@ import javax.ws.rs.PathParam; import javax.ws.rs.QueryParam; import org.apache.twill.filesystem.Location; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * {@link io.cdap.http.HttpHandler} for managing application lifecycle. */ @Singleton @Path(Constants.Gateway.API_VERSION_3 + "/namespaces/{namespace-id}") -public class AppLifecycleHttpHandler extends AbstractAppFabricHttpHandler { - - // Gson for writing response - private static final Gson GSON = new Gson(); - // Gson for decoding request - private static final Gson DECODE_GSON = new GsonBuilder() - .registerTypeAdapterFactory(new CaseInsensitiveEnumTypeAdapterFactory()) - .create(); - private static final Logger LOG = LoggerFactory.getLogger(AppLifecycleHttpHandler.class); +public class AppLifecycleHttpHandler extends AbstractAppLifecycleHttpHandler { /** * Key in json paginated applications list response. */ @@ -150,13 +128,7 @@ public class AppLifecycleHttpHandler extends AbstractAppFabricHttpHandler { /** * Runtime program service for running and managing programs. */ - private final ProgramRuntimeService runtimeService; - - private final CConfiguration configuration; - private final NamespaceQueryAdmin namespaceQueryAdmin; private final NamespacePathLocator namespacePathLocator; - private final ApplicationLifecycleService applicationLifecycleService; - private final File tmpDir; private final AccessEnforcer accessEnforcer; private final AuthenticationContext authenticationContext; private final FeatureFlagsProvider featureFlagsProvider; @@ -169,13 +141,8 @@ public class AppLifecycleHttpHandler extends AbstractAppFabricHttpHandler { ApplicationLifecycleService applicationLifecycleService, AccessEnforcer accessEnforcer, AuthenticationContext authenticationContext) { - this.configuration = configuration; - this.namespaceQueryAdmin = namespaceQueryAdmin; - this.runtimeService = runtimeService; + super(configuration, namespaceQueryAdmin, runtimeService, applicationLifecycleService); this.namespacePathLocator = namespacePathLocator; - this.applicationLifecycleService = applicationLifecycleService; - this.tmpDir = new File(new File(configuration.get(Constants.CFG_LOCAL_DATA_DIR)), - configuration.get(Constants.AppFabric.TEMP_DIR)).getAbsoluteFile(); this.accessEnforcer = accessEnforcer; this.authenticationContext = authenticationContext; this.featureFlagsProvider = new DefaultFeatureFlagsProvider(configuration); @@ -775,60 +742,7 @@ private BodyConsumer deployAppFromArtifact(final ApplicationId appId) throws IOE appId.getParent(), applicationLifecycleService.decodeUserId(authenticationContext)); // createTempFile() needs a prefix of at least 3 characters - return new AbstractBodyConsumer(File.createTempFile("apprequest-" + appId, ".json", tmpDir)) { - - @Override - protected void onFinish(HttpResponder responder, File uploadedFile) { - try (FileReader fileReader = new FileReader(uploadedFile)) { - AppRequest appRequest = DECODE_GSON.fromJson(fileReader, AppRequest.class); - - try { - ApplicationWithPrograms app = applicationLifecycleService.deployApp(appId, appRequest, - null, createProgramTerminator()); - LOG.info( - "Successfully deployed app {} in namespace {} from artifact {} with configuration {} and " - - + "principal {}", app.getApplicationId().getApplication(), - app.getApplicationId().getNamespace(), - app.getArtifactId(), appRequest.getConfig(), app.getOwnerPrincipal()); - - responder.sendJson(HttpResponseStatus.OK, GSON.toJson(getApplicationRecord(app))); - } catch (DatasetManagementException e) { - if (e.getCause() instanceof UnauthorizedException) { - throw (UnauthorizedException) e.getCause(); - } else { - throw e; - } - } - } catch (ArtifactNotFoundException e) { - responder.sendString(HttpResponseStatus.NOT_FOUND, e.getMessage()); - } catch (ConflictException e) { - responder.sendString(HttpResponseStatus.CONFLICT, e.getMessage()); - } catch (UnauthorizedException e) { - responder.sendString(HttpResponseStatus.FORBIDDEN, e.getMessage()); - } catch (InvalidArtifactException e) { - responder.sendString(HttpResponseStatus.BAD_REQUEST, e.getMessage()); - } catch (IOException e) { - LOG.error("Error reading request body for creating app {}.", appId, e); - responder.sendString(HttpResponseStatus.INTERNAL_SERVER_ERROR, String.format( - "Error while reading json request body for app %s.", appId)); - } catch (Exception e) { - LOG.error("Deploy failure", e); - responder.sendString(HttpResponseStatus.BAD_REQUEST, e.getMessage()); - } - } - }; - } - - private ApplicationRecord getApplicationRecord(ApplicationWithPrograms deployedApp) { - return new ApplicationRecord( - ArtifactSummary.from(deployedApp.getArtifactId().toApiArtifactId()), - deployedApp.getApplicationId().getApplication(), - deployedApp.getApplicationId().getVersion(), - deployedApp.getSpecification().getDescription(), - Optional.ofNullable(deployedApp.getOwnerPrincipal()).map(KerberosPrincipalId::getPrincipal) - .orElse(null), - deployedApp.getChangeDetail(), null); + return deployAppFromArtifact(appId, false); } private BodyConsumer deployApplication(final HttpResponder responder, @@ -927,54 +841,6 @@ finalOwnerPrincipalId, createProgramTerminator(), }; } - private ProgramTerminator createProgramTerminator() { - return programId -> { - switch (programId.getType()) { - case SERVICE: - case WORKER: - killProgramIfRunning(programId); - break; - } - }; - } - - private void killProgramIfRunning(ProgramId programId) { - ProgramRuntimeService.RuntimeInfo programRunInfo = findRuntimeInfo(programId, runtimeService); - if (programRunInfo != null) { - ProgramController controller = programRunInfo.getController(); - controller.kill(); - } - } - - private NamespaceId validateNamespace(@Nullable String namespace) - throws BadRequestException, NamespaceNotFoundException, AccessException { - - if (namespace == null) { - throw new BadRequestException("Path parameter namespace-id cannot be empty"); - } - - NamespaceId namespaceId; - try { - namespaceId = new NamespaceId(namespace); - } catch (IllegalArgumentException e) { - throw new BadRequestException(String.format("Invalid namespace '%s'", namespace), e); - } - - try { - if (!namespaceId.equals(NamespaceId.SYSTEM)) { - namespaceQueryAdmin.get(namespaceId); - } - } catch (NamespaceNotFoundException | AccessException e) { - throw e; - } catch (Exception e) { - // This can only happen when NamespaceAdmin uses HTTP calls to interact with namespaces. - // In AppFabricServer, NamespaceAdmin is bound to DefaultNamespaceAdmin, which interacts directly with the MDS. - // Hence, this exception will never be thrown - throw Throwables.propagate(e); - } - return namespaceId; - } - private void validateApplicationId(@Nullable String namespace, @Nullable String appId) throws BadRequestException, NamespaceNotFoundException, AccessException { validateApplicationId(validateNamespace(namespace), appId); @@ -991,21 +857,4 @@ private ApplicationId validateApplicationVersionId(@Nullable String namespace, throws BadRequestException, NamespaceNotFoundException, AccessException { return validateApplicationVersionId(validateNamespace(namespace), appId, versionId); } - - private ApplicationId validateApplicationVersionId(NamespaceId namespaceId, String appId, - String versionId) throws BadRequestException { - if (appId == null) { - throw new BadRequestException("Path parameter app-id cannot be empty"); - } - if (!EntityId.isValidId(appId)) { - throw new BadRequestException(String.format("Invalid app name '%s'", appId)); - } - if (versionId == null) { - throw new BadRequestException("Path parameter version-id cannot be empty"); - } - if (EntityId.isValidVersionId(versionId)) { - return namespaceId.app(appId, versionId); - } - throw new BadRequestException(String.format("Invalid version '%s'", versionId)); - } } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/AppLifecycleHttpHandlerInternal.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/AppLifecycleHttpHandlerInternal.java index a1170bfcb7db..60719b66e4d4 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/AppLifecycleHttpHandlerInternal.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/AppLifecycleHttpHandlerInternal.java @@ -17,25 +17,37 @@ package io.cdap.cdap.gateway.handlers; import com.google.common.collect.Iterables; -import com.google.gson.Gson; +import com.google.gson.JsonSyntaxException; import com.google.inject.Inject; import com.google.inject.Singleton; +import io.cdap.cdap.api.feature.FeatureFlagsProvider; +import io.cdap.cdap.api.security.AccessException; +import io.cdap.cdap.app.runtime.ProgramRuntimeService; import io.cdap.cdap.app.store.ApplicationFilter; import io.cdap.cdap.app.store.ScanApplicationsRequest; +import io.cdap.cdap.common.BadRequestException; import io.cdap.cdap.common.NamespaceNotFoundException; +import io.cdap.cdap.common.app.RunIds; +import io.cdap.cdap.common.conf.CConfiguration; import io.cdap.cdap.common.conf.Constants; +import io.cdap.cdap.common.feature.DefaultFeatureFlagsProvider; import io.cdap.cdap.common.namespace.NamespaceQueryAdmin; -import io.cdap.cdap.gateway.handlers.util.AbstractAppFabricHttpHandler; +import io.cdap.cdap.common.security.AuditDetail; +import io.cdap.cdap.common.security.AuditPolicy; +import io.cdap.cdap.features.Feature; import io.cdap.cdap.internal.app.services.ApplicationLifecycleService; import io.cdap.cdap.proto.ApplicationDetail; import io.cdap.cdap.proto.ApplicationRecord; +import io.cdap.cdap.proto.app.MarkLatestAppsRequest; import io.cdap.cdap.proto.id.ApplicationId; import io.cdap.cdap.proto.id.ApplicationReference; import io.cdap.cdap.proto.id.EntityId; import io.cdap.cdap.proto.id.NamespaceId; import io.cdap.cdap.spi.data.SortOrder; +import io.cdap.http.BodyConsumer; import io.cdap.http.HttpHandler; import io.cdap.http.HttpResponder; +import io.netty.handler.codec.http.FullHttpRequest; import io.netty.handler.codec.http.HttpRequest; import io.netty.handler.codec.http.HttpResponse; import io.netty.handler.codec.http.HttpResponseStatus; @@ -44,6 +56,8 @@ import java.util.Optional; import java.util.concurrent.atomic.AtomicReference; import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.PUT; import javax.ws.rs.Path; import javax.ws.rs.PathParam; import javax.ws.rs.QueryParam; @@ -54,19 +68,20 @@ */ @Singleton @Path(Constants.Gateway.INTERNAL_API_VERSION_3 + "/namespaces/{namespace-id}") -public class AppLifecycleHttpHandlerInternal extends AbstractAppFabricHttpHandler { +public class AppLifecycleHttpHandlerInternal extends AbstractAppLifecycleHttpHandler { private static final String APP_LIST_PAGINATED_KEY = "applications"; - private static final Gson GSON = new Gson(); - private final NamespaceQueryAdmin namespaceQueryAdmin; - private final ApplicationLifecycleService applicationLifecycleService; + private final FeatureFlagsProvider featureFlagsProvider; @Inject - AppLifecycleHttpHandlerInternal(NamespaceQueryAdmin namespaceQueryAdmin, + AppLifecycleHttpHandlerInternal( + CConfiguration configuration, + NamespaceQueryAdmin namespaceQueryAdmin, + ProgramRuntimeService runtimeService, ApplicationLifecycleService applicationLifecycleService) { - this.namespaceQueryAdmin = namespaceQueryAdmin; - this.applicationLifecycleService = applicationLifecycleService; + super(configuration, namespaceQueryAdmin, runtimeService, applicationLifecycleService); + this.featureFlagsProvider = new DefaultFeatureFlagsProvider(configuration); } /** @@ -200,4 +215,77 @@ public void getAppDetailForVersion(HttpRequest request, HttpResponder responder, : applicationLifecycleService.getAppDetail(appId); responder.sendJson(HttpResponseStatus.OK, GSON.toJson(appDetail)); } + + /** + * Deploy an application. This is similar to the public API to deploy an application. + * This differs from the public API by supporting the skipMakingLatest parameter. + * This behaviour (skipMakingLatest) should not be exposed in the public API and should + * only be used internally when needed. + * + * @param request {@link HttpRequest} + * @param responder {@link HttpResponder} + * @param namespaceId of the namespace where the app is to be deployed + * @param appId of the app + * @param skipMarkingLatest if true, the app will be deployed but not marked latest + * @return {@link BodyConsumer} + * @throws BadRequestException when the request params or body are not valid + * @throws NamespaceNotFoundException when the namespace is not found + * @throws AccessException in case of any security issues + * @throws UnsupportedOperationException when PIPELINE_PROMOTION_TO_PRODUCTION feature is not enabled + */ + @PUT + @Path("/apps/{app-id}") + @AuditPolicy(AuditDetail.REQUEST_BODY) + public BodyConsumer create(HttpRequest request, HttpResponder responder, + @PathParam("namespace-id") final String namespaceId, + @PathParam("app-id") final String appId, + @QueryParam("skipMarkingLatest") final boolean skipMarkingLatest) throws Exception { + if (!Feature.PIPELINE_PROMOTION_TO_PRODUCTION.isEnabled(featureFlagsProvider)) { + throw new UnsupportedOperationException("Deploying applications without marking latest is not supported"); + } + + String versionId = ApplicationId.DEFAULT_VERSION; + if (Feature.LIFECYCLE_MANAGEMENT_EDIT.isEnabled(featureFlagsProvider)) { + versionId = RunIds.generate().getId(); + } + ApplicationId applicationId = validateApplicationVersionId(validateNamespace(namespaceId), appId, versionId); + + return deployAppFromArtifact(applicationId, skipMarkingLatest); + } + + /** + * Mark provided application versions as latest. + * + * @param request {@link FullHttpRequest} + * @param responder {@link HttpResponse} + * @param namespace the namespace where the applications are deployed + * @throws Exception if either namespace or any of the application (versions) doesn't exist + */ + @POST + @Path("/apps/markLatest") + public void markApplicationsAsLatest(FullHttpRequest request, HttpResponder responder, + @PathParam("namespace-id") final String namespace) throws Exception { + if (!Feature.PIPELINE_PROMOTION_TO_PRODUCTION.isEnabled(featureFlagsProvider)) { + throw new UnsupportedOperationException("Marking applications latest is not supported."); + } + + NamespaceId namespaceId = new NamespaceId(namespace); + if (!namespaceQueryAdmin.exists(namespaceId)) { + throw new NamespaceNotFoundException(namespaceId); + } + + MarkLatestAppsRequest appsMarkLatestRequest; + try { + appsMarkLatestRequest = parseBody(request, MarkLatestAppsRequest.class); + } catch (JsonSyntaxException e) { + throw new BadRequestException("Invalid request body", e); + } + + if (appsMarkLatestRequest == null) { + throw new BadRequestException("Invalid request body."); + } + + applicationLifecycleService.markAppsAsLatest(namespaceId, appsMarkLatestRequest); + responder.sendString(HttpResponseStatus.OK, ""); + } } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/deploy/pipeline/AppDeploymentInfo.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/deploy/pipeline/AppDeploymentInfo.java index 43483ac8e060..85c46717dfed 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/deploy/pipeline/AppDeploymentInfo.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/deploy/pipeline/AppDeploymentInfo.java @@ -59,6 +59,7 @@ public class AppDeploymentInfo { @Nullable private final ApplicationSpecification deployedApplicationSpec; private final boolean isUpgrade; + private final boolean skipMarkingLatest; /** * Creates a new {@link Builder}. @@ -86,6 +87,7 @@ public static Builder copyFrom(AppDeploymentInfo other) { .setChangeDetail(other.changeDetail) .setSourceControlMeta(other.sourceControlMeta) .setIsUpgrade(other.isUpgrade) + .setSkipMarkingLatest(other.skipMarkingLatest) .setDeployedApplicationSpec(other.deployedApplicationSpec); } @@ -95,7 +97,7 @@ private AppDeploymentInfo(ArtifactId artifactId, Location artifactLocation, @Nullable String configString, @Nullable KerberosPrincipalId ownerPrincipal, boolean updateSchedules, @Nullable AppDeploymentRuntimeInfo runtimeInfo, @Nullable ChangeDetail changeDetail, @Nullable SourceControlMeta sourceControlMeta, - boolean isUpgrade, @Nullable ApplicationSpecification deployedApplicationSpec) { + boolean isUpgrade, @Nullable ApplicationSpecification deployedApplicationSpec, boolean skipMarkingLatest) { this.artifactId = artifactId; this.artifactLocation = artifactLocation; this.namespaceId = namespaceId; @@ -109,6 +111,7 @@ private AppDeploymentInfo(ArtifactId artifactId, Location artifactLocation, this.changeDetail = changeDetail; this.sourceControlMeta = sourceControlMeta; this.isUpgrade = isUpgrade; + this.skipMarkingLatest = skipMarkingLatest; this.deployedApplicationSpec = deployedApplicationSpec; } @@ -212,6 +215,10 @@ public boolean isUpgrade() { return isUpgrade; } + public boolean isSkipMarkingLatest() { + return skipMarkingLatest; + } + /** * Returns the previously deployed Application Specification. Will be null for the 1st deployment */ @@ -243,6 +250,7 @@ public static final class Builder { @Nullable private ApplicationSpecification deployedApplicationSpec; private boolean isUpgrade; + private boolean skipMarkingLatest; private Builder() { // Only for the builder() method to use @@ -320,6 +328,11 @@ public Builder setIsUpgrade(boolean isUpgrade) { return this; } + public Builder setSkipMarkingLatest(boolean skipMarkingLatest) { + this.skipMarkingLatest = skipMarkingLatest; + return this; + } + public Builder setDeployedApplicationSpec( @Nullable ApplicationSpecification deployedApplicationSpec) { this.deployedApplicationSpec = deployedApplicationSpec; @@ -341,7 +354,7 @@ public AppDeploymentInfo build() { } return new AppDeploymentInfo(artifactId, artifactLocation, namespaceId, applicationClass, appName, appVersion, configString, ownerPrincipal, updateSchedules, runtimeInfo, - changeDetail, sourceControlMeta, isUpgrade, deployedApplicationSpec); + changeDetail, sourceControlMeta, isUpgrade, deployedApplicationSpec, skipMarkingLatest); } } } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/deploy/pipeline/ApplicationDeployable.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/deploy/pipeline/ApplicationDeployable.java index 854f235c47ff..382e4be9b640 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/deploy/pipeline/ApplicationDeployable.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/deploy/pipeline/ApplicationDeployable.java @@ -26,6 +26,7 @@ import io.cdap.cdap.proto.id.ArtifactId; import io.cdap.cdap.proto.id.KerberosPrincipalId; import io.cdap.cdap.proto.sourcecontrol.SourceControlMeta; +import io.cdap.cdap.security.impersonation.UGIProvider; import io.cdap.cdap.spi.data.table.StructuredTableSpecification; import java.util.Collection; import java.util.Collections; @@ -56,6 +57,7 @@ public class ApplicationDeployable { @Nullable private final SourceControlMeta sourceControlMeta; private final boolean isUpgrade; + private final boolean skipMarkingLatest; public ApplicationDeployable(ArtifactId artifactId, Location artifactLocation, ApplicationId applicationId, ApplicationSpecification specification, @@ -65,7 +67,7 @@ public ApplicationDeployable(ArtifactId artifactId, Location artifactLocation, this(artifactId, artifactLocation, applicationId, specification, existingAppSpec, applicationDeployScope, applicationClass, null, true, Collections.emptyList(), Collections.emptyMap(), - null, null, false); + null, null, false, false); } public ApplicationDeployable(ArtifactId artifactId, Location artifactLocation, @@ -77,7 +79,7 @@ public ApplicationDeployable(ArtifactId artifactId, Location artifactLocation, boolean updateSchedules, Collection systemTables, Map metadata, @Nullable ChangeDetail changeDetail, - @Nullable SourceControlMeta sourceControlMeta, boolean isUpgrade) { + @Nullable SourceControlMeta sourceControlMeta, boolean isUpgrade, boolean skipMarkingLatest) { this.artifactId = artifactId; this.artifactLocation = artifactLocation; this.applicationId = applicationId; @@ -92,6 +94,7 @@ public ApplicationDeployable(ArtifactId artifactId, Location artifactLocation, this.changeDetail = changeDetail; this.sourceControlMeta = sourceControlMeta; this.isUpgrade = isUpgrade; + this.skipMarkingLatest = skipMarkingLatest; } /** @@ -186,6 +189,13 @@ public boolean isUpgrade() { return isUpgrade; } + /** + * Returns true if the application is not to be marked as latest. + */ + public boolean isSkipMarkingLatest() { + return skipMarkingLatest; + } + /** * Returns the source control metadata */ diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/deploy/pipeline/ApplicationRegistrationStage.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/deploy/pipeline/ApplicationRegistrationStage.java index 3d15da24e6cc..c58b32f7b3f2 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/deploy/pipeline/ApplicationRegistrationStage.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/deploy/pipeline/ApplicationRegistrationStage.java @@ -67,11 +67,15 @@ public void process(ApplicationWithPrograms input) throws Exception { boolean ownerAdded = addOwnerIfRequired(input, allAppVersionsAppIds); ApplicationMeta appMeta = new ApplicationMeta(applicationSpecification.getName(), input.getSpecification(), - input.getChangeDetail(), input.getSourceControlMeta()); + input.getChangeDetail(), input.getSourceControlMeta(), !input.isSkipMarkingLatest()); try { int editCount = store.addApplication(input.getApplicationId(), appMeta); - // increment metric : app.deploy.event.count.upgrade - if (input.isUpgrade()) { + + if (input.isSkipMarkingLatest()) { + // TODO [CDAP-20848] + // do not emit any metrics. the application may be cleaned up or marked latest later + } else if (input.isUpgrade()) { + // increment metric : app.deploy.event.count.upgrade emitMetrics(applicationId.getNamespace(), applicationId.getApplication(), Constants.Metrics.AppMetadataStore.DEPLOY_UPGRADE_COUNT); } else if (editCount == 1) { diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/deploy/pipeline/ApplicationWithPrograms.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/deploy/pipeline/ApplicationWithPrograms.java index c5710bea58bc..d814e946c964 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/deploy/pipeline/ApplicationWithPrograms.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/deploy/pipeline/ApplicationWithPrograms.java @@ -37,7 +37,8 @@ public ApplicationWithPrograms(ApplicationDeployable applicationDeployable, applicationDeployable.getApplicationClass(), applicationDeployable.getOwnerPrincipal(), applicationDeployable.canUpdateSchedules(), applicationDeployable.getSystemTables(), applicationDeployable.getMetadata(), applicationDeployable.getChangeDetail(), - applicationDeployable.getSourceControlMeta(), applicationDeployable.isUpgrade()); + applicationDeployable.getSourceControlMeta(), applicationDeployable.isUpgrade(), + applicationDeployable.isSkipMarkingLatest()); this.programDescriptors = ImmutableList.copyOf(programDescriptors); } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/deploy/pipeline/LocalArtifactLoaderStage.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/deploy/pipeline/LocalArtifactLoaderStage.java index da047b4cae34..322d41d8fea1 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/deploy/pipeline/LocalArtifactLoaderStage.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/deploy/pipeline/LocalArtifactLoaderStage.java @@ -126,6 +126,7 @@ public void process(AppDeploymentInfo deploymentInfo) throws Exception { ApplicationDeployScope.USER, deploymentInfo.getApplicationClass(), deploymentInfo.getOwnerPrincipal(), deploymentInfo.canUpdateSchedules(), appSpecInfo.getSystemTables(), metadatas, deploymentInfo.getChangeDetail(), - deploymentInfo.getSourceControlMeta(), deploymentInfo.isUpgrade())); + deploymentInfo.getSourceControlMeta(), deploymentInfo.isUpgrade(), + deploymentInfo.isSkipMarkingLatest())); } } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/preview/DefaultPreviewRunner.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/preview/DefaultPreviewRunner.java index 91ce6e0f9997..5d78e0885def 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/preview/DefaultPreviewRunner.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/preview/DefaultPreviewRunner.java @@ -193,7 +193,7 @@ public Future startPreview(PreviewRequest previewRequest) throws preview.getVersion(), artifactSummary, config, request.getChange(), null, NOOP_PROGRAM_TERMINATOR, null, request.canUpdateSchedules(), - true, userProps); + true, false, userProps); } catch (Exception e) { PreviewStatus previewStatus = new PreviewStatus(PreviewStatus.Status.DEPLOY_FAILED, submitTimeMillis, diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/ApplicationLifecycleService.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/ApplicationLifecycleService.java index bf579794b113..febfe4b589a1 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/ApplicationLifecycleService.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/ApplicationLifecycleService.java @@ -55,6 +55,7 @@ import io.cdap.cdap.common.ApplicationNotFoundException; import io.cdap.cdap.common.ArtifactAlreadyExistsException; import io.cdap.cdap.common.ArtifactNotFoundException; +import io.cdap.cdap.common.BadRequestException; import io.cdap.cdap.common.CannotBeDeletedException; import io.cdap.cdap.common.InvalidArtifactException; import io.cdap.cdap.common.NotFoundException; @@ -90,6 +91,8 @@ import io.cdap.cdap.proto.ApplicationDetail; import io.cdap.cdap.proto.PluginInstanceDetail; import io.cdap.cdap.proto.ProgramType; +import io.cdap.cdap.proto.app.AppVersion; +import io.cdap.cdap.proto.app.MarkLatestAppsRequest; import io.cdap.cdap.proto.artifact.AppRequest; import io.cdap.cdap.proto.artifact.ArtifactSortOrder; import io.cdap.cdap.proto.artifact.ChangeDetail; @@ -826,7 +829,7 @@ private ApplicationId updateApplicationInternal(ApplicationId appId, * updating an app. For example, if an update removes a flow, the terminator defines how to * stop that flow. * @return information about the deployed application - * @throws InvalidArtifactException the the artifact is invalid. For example, if it does not + * @throws InvalidArtifactException the artifact is invalid. For example, if it does not * contain any app classes * @throws ArtifactAlreadyExistsException if the specified artifact already exists * @throws IOException if there was an IO error writing the artifact @@ -848,7 +851,7 @@ public ApplicationWithPrograms deployAppAndArtifact(NamespaceId namespace, } return deployApp(namespace, appName, appVersion, configStr, null, null, programTerminator, artifactDetail, - ownerPrincipal, updateSchedules, false, Collections.emptyMap()); + ownerPrincipal, updateSchedules, false, false, Collections.emptyMap()); } catch (Exception e) { // if we added the artifact, but failed to deploy the application, delete the artifact to bring us back // to the state we were in before this call. @@ -938,7 +941,7 @@ public ApplicationWithPrograms deployApp(NamespaceId namespace, @Nullable String programTerminator, artifactDetail, ownerPrincipal, updateSchedules == null ? appUpdateSchedules : updateSchedules, - false, Collections.emptyMap()); + false, false, Collections.emptyMap()); } /** @@ -965,6 +968,7 @@ public ApplicationWithPrograms deployApp(NamespaceId namespace, @Nullable String * @param updateSchedules specifies if schedules of the workflow have to be updated, if null * value specified by the property "app.deploy.update.schedules" will be used. * @param isPreview whether the app deployment is for preview + * @param skipMarkingLatest if true, the deployed app is not marked as latest * @param userProps the user properties for the app deployment, this is basically used for * preview deployment * @return information about the deployed application @@ -983,6 +987,7 @@ public ApplicationWithPrograms deployApp(NamespaceId namespace, @Nullable String ProgramTerminator programTerminator, @Nullable KerberosPrincipalId ownerPrincipal, @Nullable Boolean updateSchedules, boolean isPreview, + boolean skipMarkingLatest, Map userProps) throws Exception { // TODO CDAP-19828 - remove appVersion parameter from method signature @@ -1000,7 +1005,7 @@ public ApplicationWithPrograms deployApp(NamespaceId namespace, @Nullable String return deployApp(namespace, appName, appVersion, configStr, changeSummary, sourceControlMeta, programTerminator, artifactDetail.get(0), ownerPrincipal, updateSchedules == null - ? appUpdateSchedules : updateSchedules, isPreview, userProps); + ? appUpdateSchedules : updateSchedules, isPreview, skipMarkingLatest, userProps); } /** @@ -1018,6 +1023,8 @@ public ApplicationWithPrograms deployApp(NamespaceId namespace, @Nullable String * @param programTerminator a program terminator that will stop programs that are removed when * updating an app. For example, if an update removes a flow, the terminator defines how to * stop that flow. + * @param skipMarkingLatest boolean, if true that app is deployed but the deployed verison is + * not marked latest * @return {@link ApplicationWithPrograms} * @throws InvalidArtifactException if the artifact does not contain any application classes * @throws IOException if there was an IO error reading artifact detail from the meta store @@ -1028,7 +1035,7 @@ public ApplicationWithPrograms deployApp(NamespaceId namespace, @Nullable String public ApplicationWithPrograms deployApp(ApplicationId appId, AppRequest appRequest, @Nullable SourceControlMeta sourceControlMeta, - ProgramTerminator programTerminator) throws Exception { + ProgramTerminator programTerminator, boolean skipMarkingLatest) throws Exception { ArtifactSummary artifactSummary = appRequest.getArtifact(); KerberosPrincipalId ownerPrincipalId = @@ -1036,7 +1043,7 @@ public ApplicationWithPrograms deployApp(ApplicationId appId, : new KerberosPrincipalId(appRequest.getOwnerPrincipal()); // if we don't null check, it gets serialized to "null". The instanceof check is also needed otherwise it causes - // unnecessary json serialization and invalid json format error. + // unnecessary json serialization and invalid json format error. Object config = appRequest.getConfig(); String configString = config == null ? null : config instanceof String ? (String) config : GSON.toJson(config); @@ -1046,7 +1053,7 @@ public ApplicationWithPrograms deployApp(ApplicationId appId, return deployApp(appId.getParent(), appId.getApplication(), appId.getVersion(), artifactSummary, configString, changeSummary, sourceControlMeta, programTerminator, ownerPrincipalId, - appRequest.canUpdateSchedules(), false, Collections.emptyMap()); + appRequest.canUpdateSchedules(), false, skipMarkingLatest, Collections.emptyMap()); } private ApplicationWithPrograms deployApp(NamespaceId namespaceId, @Nullable String appName, @@ -1058,6 +1065,7 @@ private ApplicationWithPrograms deployApp(NamespaceId namespaceId, @Nullable Str ArtifactDetail artifactDetail, @Nullable KerberosPrincipalId ownerPrincipal, boolean updateSchedules, boolean isPreview, + boolean skipMarkingLatest, Map userProps) throws Exception { // Now to deploy an app, we need ADMIN privilege on the owner principal if it is present, and also ADMIN on the app // But since at this point, app name is unknown to us, so the enforcement on the app is happening in the deploy @@ -1109,6 +1117,7 @@ private ApplicationWithPrograms deployApp(NamespaceId namespaceId, @Nullable Str : null) .setChangeDetail(change) .setSourceControlMeta(sourceControlMeta) + .setSkipMarkingLatest(skipMarkingLatest) .build(); Manager manager = managerFactory.create( @@ -1527,4 +1536,40 @@ private void emitTimeMetrics(String namespace, String appName, String metricName long timeTaken = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTime); metricsCollectionService.getContext(tags).gauge(metricName, timeTaken); } + + /** + * Marks a given list of application versions as latest. + * + * @param namespace {@link NamespaceId} of the namespace where the apps are deployed + * @param appsRequest the request object containing list of applications that need to be marked latest + * @throws BadRequestException when the list contains multiple versions of the same application + * @throws IOException when the update operation fails for any reason. + * @throws ApplicationNotFoundException when any of the applications is not found + */ + public void markAppsAsLatest(NamespaceId namespace, MarkLatestAppsRequest appsRequest) + throws BadRequestException, IOException, ApplicationNotFoundException { + + List appIds = new ArrayList<>(); + Set seenApps = new HashSet<>(); + + for (AppVersion appRequest : appsRequest.getApps()) { + // Validate the appIds do not contain duplicates, i.e. we are not trying to mark + // multiple versions of the same application as latest + if (!seenApps.add(appRequest.getName())) { + throw new BadRequestException(String.format( + "Marking multiple versions of an application (%s) as latest is not supported.", + appRequest.getName() + )); + } + + try { + appIds.add(namespace.app(appRequest.getName(), appRequest.getAppVersion())); + } catch (IllegalArgumentException | NullPointerException e) { + throw new BadRequestException(String.format("Invalid application name (%s) or version (%s)", + appRequest.getName(), appRequest.getAppVersion()), e); + } + } + + store.markApplicationsLatest(appIds); + } } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/SourceControlManagementService.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/SourceControlManagementService.java index c98f16e45eec..aa88648918da 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/SourceControlManagementService.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/SourceControlManagementService.java @@ -263,7 +263,7 @@ public ApplicationRecord pullAndDeploy(ApplicationReference appRef) throws Excep appLifecycleService.decodeUserId(authenticationContext)); ApplicationWithPrograms app = appLifecycleService.deployApp(appId, appRequest, - sourceControlMeta, x -> { }); + sourceControlMeta, x -> { }, false); LOG.info("Successfully deployed app {} in namespace {} from artifact {} with configuration {} and " + "principal {}", app.getApplicationId().getApplication(), app.getApplicationId().getNamespace(), diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/store/AppMetadataStore.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/store/AppMetadataStore.java index 08364ee8f919..ca7baeb3bd02 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/store/AppMetadataStore.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/store/AppMetadataStore.java @@ -34,6 +34,7 @@ import io.cdap.cdap.api.workflow.WorkflowToken; import io.cdap.cdap.app.store.ApplicationFilter; import io.cdap.cdap.app.store.ScanApplicationsRequest; +import io.cdap.cdap.common.ApplicationNotFoundException; import io.cdap.cdap.common.BadRequestException; import io.cdap.cdap.common.ConflictException; import io.cdap.cdap.common.app.RunIds; @@ -58,6 +59,7 @@ import io.cdap.cdap.proto.id.ProgramReference; import io.cdap.cdap.proto.id.ProgramRunId; import io.cdap.cdap.proto.sourcecontrol.SourceControlMeta; +import io.cdap.cdap.spi.data.InvalidFieldException; import io.cdap.cdap.spi.data.SortOrder; import io.cdap.cdap.spi.data.StructuredRow; import io.cdap.cdap.spi.data.StructuredTable; @@ -585,6 +587,47 @@ public Set filterProgramsExistence(Collection progr .collect(Collectors.toSet()); } + /** + * Marks a given application version as latest. This also unmarks the previous latest version. + * + * @param id {@link ApplicationId} + * @throws IOException when the updates fail + * @throws ApplicationNotFoundException when the application is not found + */ + public void updateLatestApplicationVersion(ApplicationId id) + throws IOException, ApplicationNotFoundException { + final StructuredTable appSpecTable = getApplicationSpecificationTable(); + + // First find and unmark the current latest version + Range latestRange = getLatestApplicationRange(id.getAppReference()); + try (CloseableIterator iterator = + appSpecTable.scan(latestRange, 1, StoreDefinition.AppMetadataStore.CREATION_TIME_FIELD, + SortOrder.DESC)) { + if (iterator.hasNext()) { + StructuredRow row = iterator.next(); + List> updateFields = getApplicationPrimaryKeys( + id.getNamespace(), + id.getApplication(), + row.getString(StoreDefinition.AppMetadataStore.VERSION_FIELD) + ); + updateFields.add(Fields.booleanField(StoreDefinition.AppMetadataStore.LATEST_FIELD, false)); + appSpecTable.update(updateFields); + } + } + + // check if the application being marked latest is already present in the table + // if not, then an ApplicationNotFoundException should be thrown + List> fields = getApplicationPrimaryKeys(id); + Optional existing = appSpecTable.read(fields); + if (!existing.isPresent()) { + throw new ApplicationNotFoundException(id); + } + + // then mark the new application version as latest + fields.add(Fields.booleanField(StoreDefinition.AppMetadataStore.LATEST_FIELD, true)); + appSpecTable.update(fields); + } + /** * Persisting a new application version in the table. * @@ -599,6 +642,8 @@ public int createApplicationVersion(ApplicationId id, ApplicationMeta appMeta) throws IOException, ConflictException { String parentVersion = Optional.ofNullable(appMeta.getChange()) .map(ChangeDetail::getParentVersion).orElse(null); + + boolean markAsLatest = appMeta.getIsLatest(); // Fetch the latest version ApplicationMeta latest = getLatest(id.getAppReference()); String latestVersion = latest == null ? null : latest.getSpec().getAppVersion(); @@ -607,10 +652,16 @@ public int createApplicationVersion(ApplicationId id, ApplicationMeta appMeta) String.format("Cannot deploy the application because parent version '%s' does not " + "match the latest version '%s'.", parentVersion, latestVersion)); } - // When the app does not exist, it is not an edit + + // if we are not going to mark the new version as latest, then we should leave the current + // latest version as latest. + // also, when latest is null, i.e. the app does not exist, then it's not an edit if (latest != null) { + final StructuredTable appSpecTable = getApplicationSpecificationTable(); List> fields = getApplicationPrimaryKeys(id.getNamespace(), id.getApplication(), latest.getSpec().getAppVersion()); + fields.add(Fields.booleanField(StoreDefinition.AppMetadataStore.LATEST_FIELD, !markAsLatest)); + // Assign a creation time if it's null for the previous latest app version // It is for the pre-6.8 application, we mark it as past version (like created 1s ago) // So it's sortable on creation time, especially when UI displays the version history for a pipeline @@ -618,14 +669,16 @@ public int createApplicationVersion(ApplicationId id, ApplicationMeta appMeta) // appMeta.getChange() should never be null in edit case fields.add(Fields.longField(StoreDefinition.AppMetadataStore.CREATION_TIME_FIELD, appMeta.getChange().getCreationTimeMillis() - 1000)); + appSpecTable.upsert(fields); + } else if (markAsLatest) { + appSpecTable.upsert(fields); } - fields.add(Fields.booleanField(StoreDefinition.AppMetadataStore.LATEST_FIELD, false)); - getApplicationSpecificationTable().upsert(fields); } + // Add a new version of the app writeApplication(id.getNamespace(), id.getApplication(), id.getVersion(), appMeta.getSpec(), appMeta.getChange(), - appMeta.getSourceControlMeta()); + appMeta.getSourceControlMeta(), markAsLatest); return getApplicationEditNumber( new ApplicationReference(id.getNamespaceId(), id.getApplication())); } @@ -634,8 +687,17 @@ public int createApplicationVersion(ApplicationId id, ApplicationMeta appMeta) void writeApplication(String namespaceId, String appId, String versionId, ApplicationSpecification spec, @Nullable ChangeDetail change, @Nullable SourceControlMeta sourceControlMeta) throws IOException { + writeApplication(namespaceId, appId, versionId, spec, change, sourceControlMeta, true); + } + + @VisibleForTesting + void writeApplication(String namespaceId, String appId, String versionId, + ApplicationSpecification spec, @Nullable ChangeDetail change, + @Nullable SourceControlMeta sourceControlMeta, boolean markAsLatest) throws IOException { writeApplicationSerialized(namespaceId, appId, versionId, - GSON.toJson(new ApplicationMeta(appId, spec, null)), change, sourceControlMeta); + GSON.toJson( + new ApplicationMeta(appId, spec, null, null, markAsLatest)), + change, sourceControlMeta, markAsLatest); updateApplicationEdit(namespaceId, appId); } @@ -2444,6 +2506,13 @@ private void writeApplicationSerialized(String namespaceId, String appId, String String serialized, @Nullable ChangeDetail change, @Nullable SourceControlMeta sourceControlMeta) throws IOException { + writeApplicationSerialized(namespaceId, appId, versionId, serialized, change, sourceControlMeta, true); + } + + private void writeApplicationSerialized(String namespaceId, String appId, String versionId, + String serialized, @Nullable ChangeDetail change, + @Nullable SourceControlMeta sourceControlMeta, boolean markAsLatest) + throws IOException { List> fields = getApplicationPrimaryKeys(namespaceId, appId, versionId); fields.add( Fields.stringField(StoreDefinition.AppMetadataStore.APPLICATION_DATA_FIELD, serialized)); @@ -2455,7 +2524,7 @@ private void writeApplicationSerialized(String namespaceId, String appId, String fields.add(Fields.stringField(StoreDefinition.AppMetadataStore.CHANGE_SUMMARY_FIELD, change.getDescription())); } - fields.add(Fields.booleanField(StoreDefinition.AppMetadataStore.LATEST_FIELD, true)); + fields.add(Fields.booleanField(StoreDefinition.AppMetadataStore.LATEST_FIELD, markAsLatest)); if (sourceControlMeta != null) { fields.add(Fields.stringField(StoreDefinition.AppMetadataStore.SOURCE_CONTROL_META, @@ -2550,7 +2619,7 @@ private ApplicationMeta decodeRow(StructuredRow row) { } else { changeDetail = new ChangeDetail(changeSummary, null, author, creationTimeMillis, latest); } - return new ApplicationMeta(id, spec, changeDetail, sourceControl); + return new ApplicationMeta(id, spec, changeDetail, sourceControl, latest); } private void writeToStructuredTableWithPrimaryKeys( diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/store/ApplicationMeta.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/store/ApplicationMeta.java index 0586c2df2179..56e8a0cac078 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/store/ApplicationMeta.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/store/ApplicationMeta.java @@ -36,13 +36,23 @@ public class ApplicationMeta { private final ChangeDetail change; @Nullable private final SourceControlMeta sourceControlMeta; + // the isLatest field does not need to be serialized in the ApplicationMetadata object + // as it's stored as a separate column in the app spec table. + private final transient boolean isLatest; public ApplicationMeta(String id, ApplicationSpecification spec, - @Nullable ChangeDetail change, @Nullable SourceControlMeta sourceControlMeta) { + @Nullable ChangeDetail change, @Nullable SourceControlMeta sourceControlMeta, + boolean isLatest) { this.id = id; this.spec = spec; this.change = change; this.sourceControlMeta = sourceControlMeta; + this.isLatest = isLatest; + } + + public ApplicationMeta(String id, ApplicationSpecification spec, + @Nullable ChangeDetail change, @Nullable SourceControlMeta sourceControlMeta) { + this(id, spec, change, sourceControlMeta, true); } public ApplicationMeta(String id, ApplicationSpecification spec, @Nullable ChangeDetail change) { @@ -67,6 +77,10 @@ public SourceControlMeta getSourceControlMeta() { return sourceControlMeta; } + public boolean getIsLatest() { + return isLatest; + } + @Override public String toString() { return Objects.toStringHelper(this) @@ -74,6 +88,7 @@ public String toString() { .add("spec", ADAPTER.toJson(spec)) .add("change", change) .add("sourceControlMeta", sourceControlMeta) + .add("isLatest", isLatest) .toString(); } } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/store/DefaultStore.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/store/DefaultStore.java index 5b344f5a7529..98207c9386e5 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/store/DefaultStore.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/store/DefaultStore.java @@ -582,6 +582,17 @@ public RunRecordDetail getRun(ProgramReference programRef, String runId) { }); } + @Override + public void markApplicationsLatest(Collection appIds) + throws IOException, ApplicationNotFoundException { + TransactionRunners.run(transactionRunner, context -> { + AppMetadataStore mds = getAppMetadataStore(context); + for (ApplicationId appId : appIds) { + mds.updateLatestApplicationVersion(appId); + } + }, IOException.class, ApplicationNotFoundException.class); + } + @Override public int addApplication(ApplicationId id, ApplicationMeta meta) throws ConflictException { return TransactionRunners.run(transactionRunner, context -> { diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/bootstrap/executor/AppCreator.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/bootstrap/executor/AppCreator.java index 54cdb794bd04..d7aa9421fb36 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/bootstrap/executor/AppCreator.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/bootstrap/executor/AppCreator.java @@ -69,7 +69,7 @@ public void execute(Arguments arguments) throws Exception { ApplicationId.DEFAULT_VERSION, artifactSummary, configString, arguments.getChange(), null, x -> { }, - ownerPrincipalId, arguments.canUpdateSchedules(), false, + ownerPrincipalId, arguments.canUpdateSchedules(), false, false, Collections.emptyMap()); } catch (NotFoundException | UnauthorizedException | InvalidArtifactException e) { // these exceptions are for sure not retry-able. It's hard to tell if the others are, so we just try retrying diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/capability/CapabilityApplier.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/capability/CapabilityApplier.java index 36a47b5068a9..b257832c66e1 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/capability/CapabilityApplier.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/capability/CapabilityApplier.java @@ -330,7 +330,7 @@ private void deployApp(SystemApplication application) throws Exception { .deployApp(applicationReference.getParent(), applicationReference.getApplication(), ApplicationId.DEFAULT_VERSION, application.getArtifact(), configString, null, null, NOOP_PROGRAM_TERMINATOR, - null, null, false, Collections.emptyMap()); + null, null, false, false, Collections.emptyMap()); } @VisibleForTesting diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/sysapp/SystemAppEnableExecutor.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/sysapp/SystemAppEnableExecutor.java index 88c68cf62089..183855547e66 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/sysapp/SystemAppEnableExecutor.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/sysapp/SystemAppEnableExecutor.java @@ -120,7 +120,7 @@ private ApplicationWithPrograms deploySystemApp(Arguments arguments) throws Exce appId.getVersion(), artifactSummary, configString, null, null, x -> { }, - ownerPrincipalId, arguments.canUpdateSchedules(), false, + ownerPrincipalId, arguments.canUpdateSchedules(), false, false, Collections.emptyMap()); } catch (UnauthorizedException | InvalidArtifactException e) { diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/ApplicationLifecycleServiceTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/ApplicationLifecycleServiceTest.java index f8f357b1ca5b..b5ed521c264c 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/ApplicationLifecycleServiceTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/ApplicationLifecycleServiceTest.java @@ -22,21 +22,27 @@ import io.cdap.cdap.AllProgramsApp; import io.cdap.cdap.AppWithProgramsUsingGuava; import io.cdap.cdap.CapabilityAppWithWorkflow; +import io.cdap.cdap.ConfigTestApp; import io.cdap.cdap.MetadataEmitApp; import io.cdap.cdap.MissingMapReduceWorkflowApp; import io.cdap.cdap.api.annotation.Requirements; import io.cdap.cdap.api.app.ApplicationSpecification; +import io.cdap.cdap.api.artifact.ArtifactSummary; import io.cdap.cdap.api.metadata.MetadataEntity; import io.cdap.cdap.api.metadata.MetadataScope; +import io.cdap.cdap.common.ApplicationNotFoundException; import io.cdap.cdap.common.ArtifactNotFoundException; +import io.cdap.cdap.common.BadRequestException; import io.cdap.cdap.common.conf.Constants; import io.cdap.cdap.common.id.Id; +import io.cdap.cdap.common.id.Id.Namespace; import io.cdap.cdap.common.io.Locations; import io.cdap.cdap.common.lang.ProgramResources; import io.cdap.cdap.common.lang.jar.BundleJarUtil; import io.cdap.cdap.common.test.AppJarHelper; import io.cdap.cdap.common.utils.Tasks; import io.cdap.cdap.data2.metadata.system.AppSystemMetadataWriter; +import io.cdap.cdap.features.Feature; import io.cdap.cdap.internal.AppFabricTestHelper; import io.cdap.cdap.internal.app.deploy.ProgramTerminator; import io.cdap.cdap.internal.app.deploy.Specifications; @@ -51,6 +57,9 @@ import io.cdap.cdap.proto.ProgramRecord; import io.cdap.cdap.proto.ProgramRunStatus; import io.cdap.cdap.proto.ProgramType; +import io.cdap.cdap.proto.app.AppVersion; +import io.cdap.cdap.proto.app.MarkLatestAppsRequest; +import io.cdap.cdap.proto.artifact.AppRequest; import io.cdap.cdap.proto.id.ApplicationId; import io.cdap.cdap.proto.id.ArtifactId; import io.cdap.cdap.proto.id.NamespaceId; @@ -59,6 +68,7 @@ import io.cdap.cdap.spi.metadata.MetadataKind; import io.cdap.cdap.spi.metadata.MetadataStorage; import io.cdap.cdap.spi.metadata.Read; +import io.cdap.common.http.HttpResponse; import io.netty.handler.codec.http.HttpResponseStatus; import java.io.File; import java.io.FileOutputStream; @@ -88,7 +98,7 @@ * */ public class ApplicationLifecycleServiceTest extends AppFabricTestBase { - + private static final String FEATURE_FLAG_PREFIX = "feature."; private static ApplicationLifecycleService applicationLifecycleService; private static LocationFactory locationFactory; private static ArtifactRepository artifactRepository; @@ -109,6 +119,16 @@ public static void stop() { AppFabricTestHelper.shutdown(); } + private void setPromotionToProdFlag(boolean promotionToProdFlag) { + cConf.setBoolean( + FEATURE_FLAG_PREFIX + Feature.PIPELINE_PROMOTION_TO_PRODUCTION.getFeatureFlagString(), promotionToProdFlag); + } + + private void setLcmEditFlag(boolean lcmFlag) { + cConf.setBoolean( + FEATURE_FLAG_PREFIX + Feature.LIFECYCLE_MANAGEMENT_EDIT.getFeatureFlagString(), lcmFlag); + } + // test that the call to deploy an artifact and application in a single step will delete the artifact // if the application could not be created @Test(expected = ArtifactNotFoundException.class) @@ -452,6 +472,111 @@ public void testCreateAppDetailsArchive() throws Exception { deleteNamespace("ns3"); } + @Test(expected = ApplicationNotFoundException.class) + public void testDeployWithoutMarkingAppAsLatest() throws Exception { + setPromotionToProdFlag(true); + String appName = "application_not_marked_latest"; + Id.Application appId = Id.Application.from(Id.Namespace.DEFAULT, appName); + Id.Artifact artifactId = Id.Artifact.from(Id.Namespace.DEFAULT, "appWithConfig", "1.0.0-SNAPSHOT"); + addAppArtifact(artifactId, ConfigTestApp.class); + HttpResponse resp = deployWithoutMarkingLatest( + appId, new AppRequest<>(ArtifactSummary.from(artifactId.toArtifactId()))); + Assert.assertEquals(200, resp.getResponseCode()); + + applicationLifecycleService.getLatestAppDetail(appId.toEntityId().getAppReference()); + setPromotionToProdFlag(false); + } + + @Test + public void testMarkAppsAsLatest() throws Exception { + setPromotionToProdFlag(true); + setLcmEditFlag(false); + + String appName = "application_to_be_marked_latest"; + Id.Application appId = Id.Application.from(Id.Namespace.DEFAULT, appName); + Id.Artifact artifactId = Id.Artifact.from(Id.Namespace.DEFAULT, "appWithConfig", "1.0.0-SNAPSHOT"); + addAppArtifact(artifactId, ConfigTestApp.class); + HttpResponse resp = deployWithoutMarkingLatest( + appId, new AppRequest<>(ArtifactSummary.from(artifactId.toArtifactId()))); + Assert.assertEquals(200, resp.getResponseCode()); + + applicationLifecycleService.markAppsAsLatest( + Namespace.DEFAULT.toEntityId(), + new MarkLatestAppsRequest( + Collections.singletonList(new AppVersion(appName, ApplicationId.DEFAULT_VERSION)))); + + ApplicationDetail appDetail = applicationLifecycleService.getLatestAppDetail(appId.toEntityId().getAppReference()); + Assert.assertNotNull(appDetail); + Assert.assertEquals(appName, appDetail.getName()); + Assert.assertEquals(ApplicationId.DEFAULT_VERSION, appDetail.getAppVersion()); + + setPromotionToProdFlag(false); + setLcmEditFlag(true); + } + + @Test(expected = BadRequestException.class) + public void testMarkAppsAsLatestWithDuplicateAppIds() throws Exception { + applicationLifecycleService.markAppsAsLatest( + Namespace.DEFAULT.toEntityId(), + new MarkLatestAppsRequest( + Arrays.asList(new AppVersion("app1", "v1"), + new AppVersion("app1", "v2")) + )); + } + + @Test(expected = BadRequestException.class) + public void testMarkAppsAsLatestWithInvalidAppId() throws Exception { + applicationLifecycleService.markAppsAsLatest( + Namespace.DEFAULT.toEntityId(), + new MarkLatestAppsRequest( + Collections.singletonList(new AppVersion("invalid app id", "v1")) + )); + } + + @Test(expected = BadRequestException.class) + public void testMarkAppsAsLatestWithNullAppId() throws Exception { + applicationLifecycleService.markAppsAsLatest( + Namespace.DEFAULT.toEntityId(), + new MarkLatestAppsRequest( + Collections.singletonList(new AppVersion(null, "v1")) + )); + } + + @Test(expected = BadRequestException.class) + public void testMarkAppsAsLatestWithNullAppVersion() throws Exception { + applicationLifecycleService.markAppsAsLatest( + Namespace.DEFAULT.toEntityId(), + new MarkLatestAppsRequest( + Collections.singletonList(new AppVersion("app", null)) + )); + } + + @Test(expected = ApplicationNotFoundException.class) + public void testMarkAppsAsLatestWithNonExistingAppVersion() throws Exception { + setPromotionToProdFlag(true); + setLcmEditFlag(false); + + String appName = "existing_app"; + Id.Application appId = Id.Application.from(Id.Namespace.DEFAULT, appName); + Id.Artifact artifactId = Id.Artifact.from(Id.Namespace.DEFAULT, "appWithConfig", "1.0.0-SNAPSHOT"); + addAppArtifact(artifactId, ConfigTestApp.class); + HttpResponse resp = deployWithoutMarkingLatest( + appId, new AppRequest<>(ArtifactSummary.from(artifactId.toArtifactId()))); + Assert.assertEquals(200, resp.getResponseCode()); + + try { + applicationLifecycleService.markAppsAsLatest( + Namespace.DEFAULT.toEntityId(), + new MarkLatestAppsRequest( + Arrays.asList(new AppVersion("non_existing_app", "v1"), + new AppVersion(appName, ApplicationId.DEFAULT_VERSION)) + )); + } finally { + setPromotionToProdFlag(false); + setLcmEditFlag(true); + } + } + private void waitForRuns(int expected, final ProgramId programId, final ProgramRunStatus status) throws Exception { Tasks.waitFor(expected, () -> getProgramRuns(Id.Program.fromEntityId(programId), status).size(), 5, TimeUnit.SECONDS); diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/SystemProgramManagementServiceTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/SystemProgramManagementServiceTest.java index c54f7d857120..054de2984f41 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/SystemProgramManagementServiceTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/SystemProgramManagementServiceTest.java @@ -123,6 +123,6 @@ private void deployTestApp() throws Exception { applicationLifecycleService.deployApp(NamespaceId.SYSTEM, APP_NAME, VERSION, summary, null, null, null, programId -> { // no-op - }, null, false, false, Collections.emptyMap()); + }, null, false, false, false, Collections.emptyMap()); } } diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/http/AppFabricTestBase.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/http/AppFabricTestBase.java index f24ebcbb3171..e0261f977e2a 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/http/AppFabricTestBase.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/http/AppFabricTestBase.java @@ -54,6 +54,7 @@ import io.cdap.cdap.common.NotFoundException; import io.cdap.cdap.common.conf.CConfiguration; import io.cdap.cdap.common.conf.Constants; +import io.cdap.cdap.common.conf.Constants.Gateway; import io.cdap.cdap.common.discovery.EndpointStrategy; import io.cdap.cdap.common.discovery.RandomEndpointStrategy; import io.cdap.cdap.common.discovery.URIScheme; @@ -200,6 +201,7 @@ public abstract class AppFabricTestBase { protected static final String VERSION1 = "1.0.0"; protected static final String VERSION2 = "2.0.0"; protected static TransactionRunner transactionRunner; + protected static CConfiguration cConf; private static Injector injector; @@ -230,7 +232,8 @@ public abstract class AppFabricTestBase { @BeforeClass public static void beforeClass() throws Throwable { - initializeAndStartServices(createBasicCConf()); + cConf = createBasicCConf(); + initializeAndStartServices(cConf); } protected static void initializeAndStartServices(CConfiguration cConf) throws Exception { @@ -544,6 +547,13 @@ protected HttpResponse deploy(Id.Application appId, return executeDeploy(HttpRequest.put(getEndPoint(deployPath).toURL()), appRequest); } + protected HttpResponse deployWithoutMarkingLatest(Id.Application appId, AppRequest appRequest) + throws Exception { + String deployPath = getVersionedInternalAPIPath( + "apps/" + appId.getId() + "?skipMarkingLatest=true", appId.getNamespaceId()); + return executeDeploy(HttpRequest.put(getEndPoint(deployPath).toURL()), appRequest); + } + protected HttpResponse deploy(ApplicationId appId, AppRequest appRequest) throws Exception { String deployPath = getVersionedAPIPath(String.format("apps/%s/versions/%s/create", appId.getApplication(), appId.getVersion()), @@ -603,6 +613,11 @@ protected String getVersionedAPIPath(String nonVersionedApiPath, String namespac return getVersionedAPIPath(nonVersionedApiPath, Constants.Gateway.API_VERSION_3_TOKEN, namespace); } + protected String getVersionedInternalAPIPath(String nonVersionedApiPath, String namespace) { + Preconditions.checkArgument(namespace != null, "Namespace cannot be null for v3internal APIs."); + return String.format("/%s/namespaces/%s/%s", Gateway.INTERNAL_API_VERSION_3_TOKEN, namespace, nonVersionedApiPath); + } + protected String getVersionedAPIPath(String nonVersionedApiPath, String version, String namespace) { if (!Constants.Gateway.API_VERSION_3_TOKEN.equals(version)) { throw new IllegalArgumentException( diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/store/DefaultStoreTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/store/DefaultStoreTest.java index 654d91db0cc3..ae0a911caf68 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/store/DefaultStoreTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/store/DefaultStoreTest.java @@ -78,6 +78,7 @@ import io.cdap.cdap.proto.id.ProgramRunId; import io.cdap.cdap.spi.data.SortOrder; import io.cdap.cdap.store.DefaultNamespaceStore; +import java.io.IOException; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -239,7 +240,7 @@ public void testWorkflowNodeState() { RunId sparkRunId = RunIds.generate(currentTime + 60); ProgramId sparkProgram = appId.spark(sparkName); - + setStartAndRunning(sparkProgram.run(sparkRunId.getId()), ImmutableMap.of(), systemArgs, artifactId); // stop the Spark program with failure @@ -517,6 +518,88 @@ public void testUpdateChangedApplication() throws ConflictException { Assert.assertEquals(FooMapReduceJob.class.getName(), spec.getMapReduce().get("mrJob3").getClassName()); } + @Test + public void testAddApplicationWithoutMarkingLatest() + throws ConflictException { + ApplicationId appId = new ApplicationId("account1", "app1"); + ApplicationMeta appMeta = new ApplicationMeta("app1", Specifications.from(new FooApp()), + new ChangeDetail(null, null, null, + System.currentTimeMillis()), null, false); + store.addApplication(appId, appMeta); + + ApplicationMeta storedMeta = store.getApplicationMetadata(appId); + Assert.assertNotNull(storedMeta); + Assert.assertFalse(storedMeta.getIsLatest()); + } + + @Test + public void testMarkApplicationsLatestWithNewApps() + throws ApplicationNotFoundException, ConflictException, IOException { + // Add 2 new applications without marking them latest + ApplicationId appId1 = new ApplicationId("account1", "app1"); + ApplicationMeta appMeta1 = new ApplicationMeta("app1", Specifications.from(new FooApp()), + new ChangeDetail(null, null, null, + System.currentTimeMillis()), null, false); + store.addApplication(appId1, appMeta1); + + ApplicationId appId2 = new ApplicationId("account1", "app2"); + ApplicationMeta appMeta2 = new ApplicationMeta("app2", Specifications.from(new FooApp()), + new ChangeDetail(null, null, null, + System.currentTimeMillis()), null, false); + store.addApplication(appId2, appMeta2); + + // Now mark them as latest in bulk + store.markApplicationsLatest(Arrays.asList(appId1, appId2)); + + ApplicationMeta storedMeta1 = store.getApplicationMetadata(appId1); + Assert.assertNotNull(storedMeta1); + Assert.assertTrue(storedMeta1.getIsLatest()); + + ApplicationMeta storedMeta2 = store.getApplicationMetadata(appId2); + Assert.assertNotNull(storedMeta2); + Assert.assertTrue(storedMeta2.getIsLatest()); + } + + @Test + public void testMarkApplicationsLatestWithExistingLatest() + throws ApplicationNotFoundException, ConflictException, IOException { + // Add an application as latest + ApplicationId appIdV1 = new ApplicationId("account1", "app"); + ApplicationMeta appMetaV1 = new ApplicationMeta("app1", Specifications.from(new FooApp()), + new ChangeDetail(null, null, null, + System.currentTimeMillis())); + store.addApplication(appIdV1, appMetaV1); + + // Add a new version of the application without marking latest + ApplicationId appIdV2 = new ApplicationId("account1", "app", "new-version"); + ApplicationMeta appMetaV2 = new ApplicationMeta("app", Specifications.from(new FooApp()), + new ChangeDetail(null, null, null, + System.currentTimeMillis()), null, false); + store.addApplication(appIdV2, appMetaV2); + + // Now mark the new version as latest + store.markApplicationsLatest(Collections.singletonList(appIdV2)); + + ApplicationMeta storedMetaV1 = store.getApplicationMetadata(appIdV1); + Assert.assertNotNull(storedMetaV1); + Assert.assertFalse(storedMetaV1.getIsLatest()); + + ApplicationMeta storedMetaV2 = store.getApplicationMetadata(appIdV2); + Assert.assertNotNull(storedMetaV2); + Assert.assertTrue(storedMetaV2.getIsLatest()); + } + + @Test(expected = ApplicationNotFoundException.class) + public void testMarkApplicationsLatestWithNonExistingApp() + throws ApplicationNotFoundException, IOException { + // Add an application as latest + ApplicationId appId = new ApplicationId("account1", "app"); + + // Now try marking this non existing app as latest + // this should throw ApplicationNotFoundException (expected in this test) + store.markApplicationsLatest(Collections.singletonList(appId)); + } + private static class FooApp extends AbstractApplication { @Override public void configure() { @@ -743,7 +826,7 @@ public void testRuntimeArgsDeletion() throws ConflictException { ImmutableMap.of("path", "/data"), new HashMap<>(), artifactId); setStartAndRunning(workflowProgramId.run(workflowRunId), ImmutableMap.of("whitelist", "cask"), new HashMap<>(), artifactId); - + ProgramRunId mapreduceProgramRunId = mapreduceProgramId.run(mapreduceRunId); ProgramRunId workflowProgramRunId = workflowProgramId.run(workflowRunId); diff --git a/cdap-features/src/main/java/io/cdap/cdap/features/Feature.java b/cdap-features/src/main/java/io/cdap/cdap/features/Feature.java index 4f15fab8a6a7..d88118274025 100644 --- a/cdap-features/src/main/java/io/cdap/cdap/features/Feature.java +++ b/cdap-features/src/main/java/io/cdap/cdap/features/Feature.java @@ -41,7 +41,8 @@ public enum Feature { WRANGLER_PRECONDITION_SQL("6.9.1"), WRANGLER_EXECUTION_SQL("6.10.0"), WRANGLER_SCHEMA_MANAGEMENT("6.10.0"), - NAMESPACED_SERVICE_ACCOUNTS("6.10.0"); + NAMESPACED_SERVICE_ACCOUNTS("6.10.0"), + PIPELINE_PROMOTION_TO_PRODUCTION("6.10.0"); private final PlatformInfo.Version versionIntroduced; private final boolean defaultAfterIntroduction; diff --git a/cdap-gateway/src/main/java/io/cdap/cdap/gateway/router/ExpectedNumberOfAuditPolicyPaths.java b/cdap-gateway/src/main/java/io/cdap/cdap/gateway/router/ExpectedNumberOfAuditPolicyPaths.java index 12d4bba58be8..d05820b1b5da 100644 --- a/cdap-gateway/src/main/java/io/cdap/cdap/gateway/router/ExpectedNumberOfAuditPolicyPaths.java +++ b/cdap-gateway/src/main/java/io/cdap/cdap/gateway/router/ExpectedNumberOfAuditPolicyPaths.java @@ -22,5 +22,5 @@ */ public final class ExpectedNumberOfAuditPolicyPaths { - public static final int EXPECTED_PATH_NUMBER = 46; + public static final int EXPECTED_PATH_NUMBER = 47; } diff --git a/cdap-proto/src/main/java/io/cdap/cdap/proto/app/AppVersion.java b/cdap-proto/src/main/java/io/cdap/cdap/proto/app/AppVersion.java new file mode 100644 index 000000000000..2fed3a6df954 --- /dev/null +++ b/cdap-proto/src/main/java/io/cdap/cdap/proto/app/AppVersion.java @@ -0,0 +1,58 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.proto.app; + +import java.util.Objects; + +/** + * Contains an app name and version. + */ +public class AppVersion { + private final String name; + private final String appVersion; + + public AppVersion(String name, String appVersion) { + this.name = name; + this.appVersion = appVersion; + } + + public String getName() { + return name; + } + + public String getAppVersion() { + return appVersion; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof AppVersion)) { + return false; + } + AppVersion that = (AppVersion) o; + return Objects.equals(name, that.name) && Objects.equals(appVersion, + that.appVersion); + } + + @Override + public int hashCode() { + return Objects.hash(name, appVersion); + } +} diff --git a/cdap-proto/src/main/java/io/cdap/cdap/proto/app/MarkLatestAppsRequest.java b/cdap-proto/src/main/java/io/cdap/cdap/proto/app/MarkLatestAppsRequest.java new file mode 100644 index 000000000000..ab25c49ed2ae --- /dev/null +++ b/cdap-proto/src/main/java/io/cdap/cdap/proto/app/MarkLatestAppsRequest.java @@ -0,0 +1,61 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.proto.app; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +/** + * Request class for the markLatest api. + */ +public class MarkLatestAppsRequest { + private final List apps; + + public MarkLatestAppsRequest(List apps) { + this.apps = apps; + } + + /** + * Get the list of apps the are to be marked latest. + * + * @return List of {@link AppVersion} or an empty list. Should not return null. + */ + public List getApps() { + if (null == apps) { + return new ArrayList<>(); + } + return apps; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof MarkLatestAppsRequest)) { + return false; + } + MarkLatestAppsRequest that = (MarkLatestAppsRequest) o; + return Objects.equals(apps, that.apps); + } + + @Override + public int hashCode() { + return Objects.hash(apps); + } +}