diff --git a/api/src/main/java/com/cloud/event/EventTypes.java b/api/src/main/java/com/cloud/event/EventTypes.java index 484dc9b001e7..67b6ce3f54c7 100644 --- a/api/src/main/java/com/cloud/event/EventTypes.java +++ b/api/src/main/java/com/cloud/event/EventTypes.java @@ -30,6 +30,7 @@ import org.apache.cloudstack.config.Configuration; import org.apache.cloudstack.ha.HAConfig; import org.apache.cloudstack.usage.Usage; +import org.apache.cloudstack.vm.schedule.VMSchedule; import com.cloud.dc.DataCenter; import com.cloud.dc.DataCenterGuestIpv6Prefix; @@ -84,7 +85,6 @@ import com.cloud.vm.Nic; import com.cloud.vm.NicSecondaryIp; import com.cloud.vm.VirtualMachine; -import org.apache.cloudstack.vm.schedule.VMSchedule; public class EventTypes { @@ -320,6 +320,7 @@ public class EventTypes { public static final String EVENT_DOMAIN_UPDATE = "DOMAIN.UPDATE"; // Snapshots + public static final String EVENT_SNAPSHOT_COPY = "SNAPSHOT.COPY"; public static final String EVENT_SNAPSHOT_CREATE = "SNAPSHOT.CREATE"; public static final String EVENT_SNAPSHOT_ON_PRIMARY = "SNAPSHOT.ON_PRIMARY"; public static final String EVENT_SNAPSHOT_OFF_PRIMARY = "SNAPSHOT.OFF_PRIMARY"; diff --git a/api/src/main/java/com/cloud/storage/VolumeApiService.java b/api/src/main/java/com/cloud/storage/VolumeApiService.java index 09a3a33d9152..8d5f7892f102 100644 --- a/api/src/main/java/com/cloud/storage/VolumeApiService.java +++ b/api/src/main/java/com/cloud/storage/VolumeApiService.java @@ -19,9 +19,9 @@ package com.cloud.storage; import java.net.MalformedURLException; +import java.util.List; import java.util.Map; -import com.cloud.utils.fsm.NoTransitionException; import org.apache.cloudstack.api.command.user.volume.AssignVolumeCmd; import org.apache.cloudstack.api.command.user.volume.AttachVolumeCmd; import org.apache.cloudstack.api.command.user.volume.ChangeOfferingForVolumeCmd; @@ -37,6 +37,7 @@ import com.cloud.exception.ResourceAllocationException; import com.cloud.user.Account; +import com.cloud.utils.fsm.NoTransitionException; public interface VolumeApiService { @@ -105,10 +106,10 @@ public interface VolumeApiService { Volume detachVolumeFromVM(DetachVolumeCmd cmd); - Snapshot takeSnapshot(Long volumeId, Long policyId, Long snapshotId, Account account, boolean quiescevm, Snapshot.LocationType locationType, boolean asyncBackup, Map tags) + Snapshot takeSnapshot(Long volumeId, Long policyId, Long snapshotId, Account account, boolean quiescevm, Snapshot.LocationType locationType, boolean asyncBackup, Map tags, List zoneIds) throws ResourceAllocationException; - Snapshot allocSnapshot(Long volumeId, Long policyId, String snapshotName, Snapshot.LocationType locationType) throws ResourceAllocationException; + Snapshot allocSnapshot(Long volumeId, Long policyId, String snapshotName, Snapshot.LocationType locationType, List zoneIds) throws ResourceAllocationException; Volume updateVolume(long volumeId, String path, String state, Long storageId, Boolean displayVolume, String customId, long owner, String chainInfo, String name); diff --git a/api/src/main/java/com/cloud/storage/snapshot/SnapshotApiService.java b/api/src/main/java/com/cloud/storage/snapshot/SnapshotApiService.java index 38e5e105a483..0893f337ce2f 100644 --- a/api/src/main/java/com/cloud/storage/snapshot/SnapshotApiService.java +++ b/api/src/main/java/com/cloud/storage/snapshot/SnapshotApiService.java @@ -18,18 +18,20 @@ import java.util.List; +import org.apache.cloudstack.api.command.user.snapshot.CopySnapshotCmd; import org.apache.cloudstack.api.command.user.snapshot.CreateSnapshotPolicyCmd; import org.apache.cloudstack.api.command.user.snapshot.DeleteSnapshotPoliciesCmd; import org.apache.cloudstack.api.command.user.snapshot.ListSnapshotPoliciesCmd; import org.apache.cloudstack.api.command.user.snapshot.ListSnapshotsCmd; +import org.apache.cloudstack.api.command.user.snapshot.UpdateSnapshotPolicyCmd; import com.cloud.api.commands.ListRecurringSnapshotScheduleCmd; import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.StorageUnavailableException; import com.cloud.storage.Snapshot; import com.cloud.storage.Volume; import com.cloud.user.Account; import com.cloud.utils.Pair; -import org.apache.cloudstack.api.command.user.snapshot.UpdateSnapshotPolicyCmd; public interface SnapshotApiService { @@ -50,7 +52,7 @@ public interface SnapshotApiService { * @param snapshotId * TODO */ - boolean deleteSnapshot(long snapshotId); + boolean deleteSnapshot(long snapshotId, Long zoneId); /** * Creates a policy with specified schedule. maxSnaps specifies the number of most recent snapshots that are to be @@ -88,7 +90,7 @@ public interface SnapshotApiService { Snapshot allocSnapshot(Long volumeId, Long policyId, String snapshotName, Snapshot.LocationType locationType) throws ResourceAllocationException; - Snapshot allocSnapshot(Long volumeId, Long policyId, String snapshotName, Snapshot.LocationType locationType, Boolean isFromVmSnapshot) + Snapshot allocSnapshot(Long volumeId, Long policyId, String snapshotName, Snapshot.LocationType locationType, Boolean isFromVmSnapshot, List zoneIds) throws ResourceAllocationException; @@ -124,4 +126,6 @@ Snapshot allocSnapshot(Long volumeId, Long policyId, String snapshotName, Snapsh SnapshotPolicy updateSnapshotPolicy(UpdateSnapshotPolicyCmd updateSnapshotPolicyCmd); void markVolumeSnapshotsAsDestroyed(Volume volume); + + Snapshot copySnapshot(CopySnapshotCmd cmd) throws StorageUnavailableException, ResourceAllocationException; } diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 3e0e65220e17..9503f9b76fb8 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -76,8 +76,10 @@ public class ApiConstants { public static final String CSR = "csr"; public static final String PRIVATE_KEY = "privatekey"; public static final String DATASTORE_HOST = "datastorehost"; + public static final String DATASTORE_ID = "datastoreid"; public static final String DATASTORE_NAME = "datastorename"; public static final String DATASTORE_PATH = "datastorepath"; + public static final String DATASTORE_STATE = "datastorestate"; public static final String DATASTORE_TYPE = "datastoretype"; public static final String DOMAIN_SUFFIX = "domainsuffix"; public static final String DNS_SEARCH_ORDER = "dnssearchorder"; @@ -492,6 +494,7 @@ public class ApiConstants { public static final String ZONE = "zone"; public static final String ZONE_ID = "zoneid"; public static final String ZONE_NAME = "zonename"; + public static final String ZONE_WISE = "zonewise"; public static final String NETWORK_TYPE = "networktype"; public static final String PAGE = "page"; public static final String PAGE_SIZE = "pagesize"; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CopySnapshotCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CopySnapshotCmd.java new file mode 100644 index 000000000000..f6d16c3eb493 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CopySnapshotCmd.java @@ -0,0 +1,181 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.user.snapshot; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseAsyncCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ResponseObject; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.command.user.UserCmd; +import org.apache.cloudstack.api.response.SnapshotResponse; +import org.apache.cloudstack.api.response.ZoneResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.commons.collections.CollectionUtils; +import org.apache.log4j.Logger; + +import com.cloud.dc.DataCenter; +import com.cloud.event.EventTypes; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.exception.StorageUnavailableException; +import com.cloud.storage.Snapshot; +import com.cloud.user.Account; + +@APICommand(name = "copySnapshot", description = "Copies a snapshot from one zone to another.", + responseObject = SnapshotResponse.class, responseView = ResponseObject.ResponseView.Restricted, + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, since = "4.19.0", + authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}) +public class CopySnapshotCmd extends BaseAsyncCmd implements UserCmd { + public static final Logger s_logger = Logger.getLogger(CopySnapshotCmd.class.getName()); + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, + entityType = SnapshotResponse.class, required = true, description = "the ID of the snapshot.") + private Long id; + + @Parameter(name = ApiConstants.SOURCE_ZONE_ID, + type = CommandType.UUID, + entityType = ZoneResponse.class, + description = "The ID of the zone in which the snapshot is currently present. " + + "If not specified then the zone of snapshot's volume will be used.") + private Long sourceZoneId; + + @Parameter(name = ApiConstants.DESTINATION_ZONE_ID, + type = CommandType.UUID, + entityType = ZoneResponse.class, + required = false, + description = "The ID of the zone the snapshot is being copied to.") + protected Long destZoneId; + + @Parameter(name = ApiConstants.DESTINATION_ZONE_ID_LIST, + type=CommandType.LIST, + collectionType = CommandType.UUID, + entityType = ZoneResponse.class, + required = false, + description = "A comma-separated list of IDs of the zones that the snapshot needs to be copied to. " + + "Specify this list if the snapshot needs to copied to multiple zones in one go. " + + "Do not specify destzoneid and destzoneids together, however one of them is required.") + protected List destZoneIds; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + + public Long getId() { + return id; + } + + public Long getSourceZoneId() { + return sourceZoneId; + } + + public List getDestinationZoneIds() { + if (destZoneIds != null && destZoneIds.size() != 0) { + return destZoneIds; + } + if (destZoneId != null) { + List < Long > destIds = new ArrayList<>(); + destIds.add(destZoneId); + return destIds; + } + return null; + } + + @Override + public String getEventType() { + return EventTypes.EVENT_SNAPSHOT_COPY; + } + + @Override + public String getEventDescription() { + StringBuilder descBuilder = new StringBuilder(); + if (getDestinationZoneIds() != null) { + for (Long destId : getDestinationZoneIds()) { + descBuilder.append(", "); + descBuilder.append(_uuidMgr.getUuid(DataCenter.class, destId)); + } + if (descBuilder.length() > 0) { + descBuilder.deleteCharAt(0); + } + } + + return "copying snapshot: " + _uuidMgr.getUuid(Snapshot.class, getId()) + ((descBuilder.length() > 0) ? " to zones: " + descBuilder.toString() : ""); + } + + @Override + public ApiCommandResourceType getApiResourceType() { + return ApiCommandResourceType.Snapshot; + } + + @Override + public Long getApiResourceId() { + return getId(); + } + + @Override + public long getEntityOwnerId() { + Snapshot snapshot = _entityMgr.findById(Snapshot.class, getId()); + if (snapshot != null) { + return snapshot.getAccountId(); + } + return Account.ACCOUNT_ID_SYSTEM; + } + + @Override + public void execute() throws ResourceUnavailableException { + try { + if (destZoneId == null && CollectionUtils.isEmpty(destZoneIds)) + throw new ServerApiException(ApiErrorCode.PARAM_ERROR, + "Either destzoneid or destzoneids parameters have to be specified."); + + if (destZoneId != null && CollectionUtils.isNotEmpty(destZoneIds)) + throw new ServerApiException(ApiErrorCode.PARAM_ERROR, + "Both destzoneid and destzoneids cannot be specified at the same time."); + + CallContext.current().setEventDetails(getEventDescription()); + Snapshot snapshot = _snapshotService.copySnapshot(this); + + if (snapshot != null) { + SnapshotResponse response = _queryService.listSnapshot(this); + response.setResponseName(getCommandName()); + setResponseObject(response); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to copy snapshot"); + } + } catch (StorageUnavailableException ex) { + s_logger.warn("Exception: ", ex); + throw new ServerApiException(ApiErrorCode.RESOURCE_UNAVAILABLE_ERROR, ex.getMessage()); + } catch (ResourceAllocationException ex) { + s_logger.warn("Exception: ", ex); + throw new ServerApiException(ApiErrorCode.RESOURCE_ALLOCATION_ERROR, ex.getMessage()); + } + + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotCmd.java index 56e112963a95..eed3aa49fa59 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotCmd.java @@ -18,6 +18,7 @@ import java.util.Collection; import java.util.HashMap; +import java.util.List; import java.util.Map; import org.apache.cloudstack.api.APICommand; @@ -32,6 +33,7 @@ import org.apache.cloudstack.api.response.SnapshotPolicyResponse; import org.apache.cloudstack.api.response.SnapshotResponse; import org.apache.cloudstack.api.response.VolumeResponse; +import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.commons.collections.MapUtils; import org.apache.log4j.Logger; @@ -90,6 +92,15 @@ public class CreateSnapshotCmd extends BaseAsyncCreateCmd { @Parameter(name = ApiConstants.TAGS, type = CommandType.MAP, description = "Map of tags (key/value pairs)") private Map tags; + @Parameter(name = ApiConstants.ZONE_ID_LIST, + type=CommandType.LIST, + collectionType = CommandType.UUID, + entityType = ZoneResponse.class, + description = "A comma-separated list of IDs of the zones in which the snapshot will be made available. " + + "The snapshot will always be made available in the zone in which the volume is present.", + since = "4.19.0") + protected List zoneIds; + private String syncObjectType = BaseAsyncCmd.snapshotHostSyncObject; // /////////////////////////////////////////////////// @@ -148,6 +159,10 @@ private Long getHostId() { return _snapshotService.getHostIdForSnapshotOperation(volume); } + public List getZoneIds() { + return zoneIds; + } + // /////////////////////////////////////////////////// // ///////////// API Implementation/////////////////// // /////////////////////////////////////////////////// @@ -196,7 +211,7 @@ public ApiCommandResourceType getApiResourceType() { @Override public void create() throws ResourceAllocationException { - Snapshot snapshot = _volumeService.allocSnapshot(getVolumeId(), getPolicyId(), getSnapshotName(), getLocationType()); + Snapshot snapshot = _volumeService.allocSnapshot(getVolumeId(), getPolicyId(), getSnapshotName(), getLocationType(), getZoneIds()); if (snapshot != null) { setEntityId(snapshot.getId()); setEntityUuid(snapshot.getUuid()); @@ -210,7 +225,7 @@ public void execute() { Snapshot snapshot; try { snapshot = - _volumeService.takeSnapshot(getVolumeId(), getPolicyId(), getEntityId(), _accountService.getAccount(getEntityOwnerId()), getQuiescevm(), getLocationType(), getAsyncBackup(), getTags()); + _volumeService.takeSnapshot(getVolumeId(), getPolicyId(), getEntityId(), _accountService.getAccount(getEntityOwnerId()), getQuiescevm(), getLocationType(), getAsyncBackup(), getTags(), getZoneIds()); if (snapshot != null) { SnapshotResponse response = _responseGenerator.createSnapshotResponse(snapshot); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotFromVMSnapshotCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotFromVMSnapshotCmd.java index 1fd7cfd013b1..7b89e87202d9 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotFromVMSnapshotCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotFromVMSnapshotCmd.java @@ -186,7 +186,7 @@ public void execute() { } finally { if (snapshot == null) { try { - _snapshotService.deleteSnapshot(getEntityId()); + _snapshotService.deleteSnapshot(getEntityId(), null); } catch (Exception e) { s_logger.debug("Failed to clean failed snapshot" + getEntityId()); } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotPolicyCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotPolicyCmd.java index a3b798405eb0..00bfb9e7e2c9 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotPolicyCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotPolicyCmd.java @@ -18,6 +18,7 @@ import java.util.Collection; import java.util.HashMap; +import java.util.List; import java.util.Map; import org.apache.cloudstack.acl.RoleType; @@ -30,6 +31,7 @@ import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.response.SnapshotPolicyResponse; import org.apache.cloudstack.api.response.VolumeResponse; +import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.commons.collections.MapUtils; import org.apache.log4j.Logger; @@ -75,6 +77,14 @@ public class CreateSnapshotPolicyCmd extends BaseCmd { @Parameter(name = ApiConstants.TAGS, type = CommandType.MAP, description = "Map of tags (key/value pairs)") private Map tags; + @Parameter(name = ApiConstants.ZONE_ID_LIST, + type=CommandType.LIST, + collectionType = CommandType.UUID, + entityType = ZoneResponse.class, + description = "A list of IDs of the zones in which the snapshots will be made available." + + "The snapshots will always be made available in the zone in which the volume is present.") + protected List zoneIds; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -107,6 +117,10 @@ public boolean isDisplay() { return display; } + public List getZoneIds() { + return zoneIds; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/DeleteSnapshotCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/DeleteSnapshotCmd.java index 8530e0ff5840..6d71b1363b42 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/DeleteSnapshotCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/DeleteSnapshotCmd.java @@ -16,6 +16,7 @@ // under the License. package org.apache.cloudstack.api.command.user.snapshot; +import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.log4j.Logger; import org.apache.cloudstack.acl.SecurityChecker.AccessType; @@ -48,6 +49,10 @@ public class DeleteSnapshotCmd extends BaseAsyncCmd { @Parameter(name=ApiConstants.ID, type=CommandType.UUID, entityType = SnapshotResponse.class, required=true, description="The ID of the snapshot") private Long id; + @Parameter(name=ApiConstants.ZONE_ID, type=CommandType.UUID, entityType = ZoneResponse.class, + description="The ID of the zone for the snapshot", since = "4.19.0") + private Long zoneId; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// @@ -57,6 +62,10 @@ public Long getId() { return id; } + public Long getZoneId() { + return zoneId; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// @@ -94,7 +103,7 @@ public Long getApiResourceId() { @Override public void execute() { CallContext.current().setEventDetails("Snapshot Id: " + this._uuidMgr.getUuid(Snapshot.class, getId())); - boolean result = _snapshotService.deleteSnapshot(getId()); + boolean result = _snapshotService.deleteSnapshot(getId(), getZoneId()); if (result) { SuccessResponse response = new SuccessResponse(getCommandName()); setResponseObject(response); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/ListSnapshotsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/ListSnapshotsCmd.java index 0b4a215733cf..23515284e4c1 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/ListSnapshotsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/ListSnapshotsCmd.java @@ -16,11 +16,8 @@ // under the License. package org.apache.cloudstack.api.command.user.snapshot; -import java.util.ArrayList; import java.util.List; -import org.apache.log4j.Logger; - import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiConstants; @@ -30,9 +27,9 @@ import org.apache.cloudstack.api.response.SnapshotResponse; import org.apache.cloudstack.api.response.VolumeResponse; import org.apache.cloudstack.api.response.ZoneResponse; +import org.apache.log4j.Logger; import com.cloud.storage.Snapshot; -import com.cloud.utils.Pair; @APICommand(name = "listSnapshots", description = "Lists all available snapshots for the account.", responseObject = SnapshotResponse.class, entityType = { Snapshot.class }, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) @@ -65,6 +62,13 @@ public class ListSnapshotsCmd extends BaseListTaggedResourcesCmd { @Parameter(name = ApiConstants.ZONE_ID, type = CommandType.UUID, entityType = ZoneResponse.class, description = "list snapshots by zone id") private Long zoneId; + @Parameter(name = ApiConstants.SHOW_UNIQUE, type = CommandType.BOOLEAN, description = "If set to false, list templates across zones and their storages", since = "4.19.0") + private Boolean showUnique; + + @Parameter(name = ApiConstants.LOCATION_TYPE, type = CommandType.STRING, description = "list snapshots by location type. Used only when showunique=false. " + + "Valid location types: 'primary', 'secondary'. Default is empty", since = "4.19.0") + private String locationType; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -93,6 +97,20 @@ public Long getZoneId() { return zoneId; } + public boolean isShowUnique() { + if (Boolean.FALSE.equals(showUnique)) { + return false; + } + return true; + } + + public String getLocationType() { + if (!isShowUnique()) { + return locationType; + } + return null; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// @@ -104,15 +122,7 @@ public ApiCommandResourceType getApiResourceType() { @Override public void execute() { - Pair, Integer> result = _snapshotService.listSnapshots(this); - ListResponse response = new ListResponse(); - List snapshotResponses = new ArrayList(); - for (Snapshot snapshot : result.first()) { - SnapshotResponse snapshotResponse = _responseGenerator.createSnapshotResponse(snapshot); - snapshotResponse.setObjectName("snapshot"); - snapshotResponses.add(snapshotResponse); - } - response.setResponses(snapshotResponses, result.second()); + ListResponse response = _queryService.listSnapshots(this); response.setResponseName(getCommandName()); setResponseObject(response); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/template/CreateTemplateCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/template/CreateTemplateCmd.java index ea4b5995bf30..73a6155c8c58 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/template/CreateTemplateCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/template/CreateTemplateCmd.java @@ -29,6 +29,7 @@ import org.apache.cloudstack.api.response.VolumeResponse; import org.apache.cloudstack.api.response.ProjectResponse; +import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; @@ -135,6 +136,9 @@ public class CreateTemplateCmd extends BaseAsyncCreateCmd implements UserCmd { @Parameter(name = ApiConstants.PROJECT_ID, type = CommandType.UUID, entityType = ProjectResponse.class, description = "create template for the project") private Long projectId; + @Parameter(name = ApiConstants.ZONE_ID, type = CommandType.UUID, entityType = ZoneResponse.class, description = "the zone for the template. Can be specified with snapshot only", since = "4.19.0") + private Long zoneId; + // /////////////////////////////////////////////////// // ///////////////// Accessors /////////////////////// // /////////////////////////////////////////////////// @@ -209,6 +213,10 @@ public boolean isDynamicallyScalable() { return isDynamicallyScalable == null ? false : isDynamicallyScalable; } + public Long getZoneId() { + return zoneId; + } + // /////////////////////////////////////////////////// // ///////////// API Implementation/////////////////// // /////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/zone/ListZonesCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/zone/ListZonesCmd.java index c79afb1c9721..c29f3a851061 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/zone/ListZonesCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/zone/ListZonesCmd.java @@ -16,10 +16,9 @@ // under the License. package org.apache.cloudstack.api.command.user.zone; +import java.util.List; import java.util.Map; -import org.apache.log4j.Logger; - import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseListCmd; @@ -30,6 +29,7 @@ import org.apache.cloudstack.api.response.DomainResponse; import org.apache.cloudstack.api.response.ListResponse; import org.apache.cloudstack.api.response.ZoneResponse; +import org.apache.log4j.Logger; @APICommand(name = "listZones", description = "Lists zones", responseObject = ZoneResponse.class, responseView = ResponseView.Restricted, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) @@ -44,6 +44,9 @@ public class ListZonesCmd extends BaseListCmd implements UserCmd { @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = ZoneResponse.class, description = "the ID of the zone") private Long id; + @Parameter(name = ApiConstants.IDS, type = CommandType.LIST, collectionType = CommandType.UUID, entityType = ZoneResponse.class, description = "the IDs of the zones, mutually exclusive with id", since = "4.19.0") + private List ids; + @Parameter(name = ApiConstants.AVAILABLE, type = CommandType.BOOLEAN, description = "true if you want to retrieve all available Zones. False if you only want to return the Zones" @@ -76,6 +79,10 @@ public Long getId() { return id; } + public List getIds() { + return ids; + } + public Boolean isAvailable() { return available; } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/SnapshotPolicyResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/SnapshotPolicyResponse.java index d1e535ee7433..bfa1cca1ca08 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/SnapshotPolicyResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/SnapshotPolicyResponse.java @@ -58,8 +58,13 @@ public class SnapshotPolicyResponse extends BaseResponseWithTagInformation { @Param(description = "is this policy for display to the regular user", since = "4.4", authorized = {RoleType.Admin}) private Boolean forDisplay; + @SerializedName(ApiConstants.ZONE) + @Param(description = "The list of zones in which snapshot backup is scheduled", responseObject = ZoneResponse.class, since = "4.19.0") + protected Set zones; + public SnapshotPolicyResponse() { tags = new LinkedHashSet(); + zones = new LinkedHashSet<>(); } public String getId() { @@ -121,4 +126,8 @@ public void setForDisplay(Boolean forDisplay) { public void setTags(Set tags) { this.tags = tags; } + + public void setZones(Set zones) { + this.zones = zones; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/SnapshotResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/SnapshotResponse.java index 5490e8a4046d..e160f64ebe91 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/SnapshotResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/SnapshotResponse.java @@ -18,6 +18,7 @@ import java.util.Date; import java.util.LinkedHashSet; +import java.util.Map; import java.util.Set; import org.apache.cloudstack.api.ApiConstants; @@ -29,7 +30,7 @@ import com.google.gson.annotations.SerializedName; @EntityReference(value = Snapshot.class) -public class SnapshotResponse extends BaseResponseWithTagInformation implements ControlledEntityResponse { +public class SnapshotResponse extends BaseResponseWithTagInformation implements ControlledViewEntityResponse { @SerializedName(ApiConstants.ID) @Param(description = "ID of the snapshot") private String id; @@ -90,6 +91,10 @@ public class SnapshotResponse extends BaseResponseWithTagInformation implements @Param(description = "the state of the snapshot. BackedUp means that snapshot is ready to be used; Creating - the snapshot is being allocated on the primary storage; BackingUp - the snapshot is being backed up on secondary storage") private Snapshot.State state; + @SerializedName(ApiConstants.STATUS) + @Param(description = "the status of the template") + private String status; + @SerializedName(ApiConstants.PHYSICAL_SIZE) @Param(description = "physical size of backedup snapshot on image store") private long physicalSize; @@ -98,6 +103,10 @@ public class SnapshotResponse extends BaseResponseWithTagInformation implements @Param(description = "id of the availability zone") private String zoneId; + @SerializedName(ApiConstants.ZONE_NAME) + @Param(description = "name of the availability zone") + private String zoneName; + @SerializedName(ApiConstants.REVERTABLE) @Param(description = "indicates whether the underlying storage supports reverting the volume to this snapshot") private boolean revertable; @@ -114,6 +123,26 @@ public class SnapshotResponse extends BaseResponseWithTagInformation implements @Param(description = "virtual size of backedup snapshot on image store") private long virtualSize; + @SerializedName(ApiConstants.DATASTORE_ID) + @Param(description = "ID of the datastore for the snapshot entry", since = "4.19.0") + private String datastoreId; + + @SerializedName(ApiConstants.DATASTORE_NAME) + @Param(description = "name of the datastore for the snapshot entry", since = "4.19.0") + private String datastoreName; + + @SerializedName(ApiConstants.DATASTORE_STATE) + @Param(description = "state of the snapshot on the datastore", since = "4.19.0") + private String datastoreState; + + @SerializedName(ApiConstants.DATASTORE_TYPE) + @Param(description = "type of the datastore for the snapshot entry", since = "4.19.0") + private String datastoreType; + + @SerializedName(ApiConstants.DOWNLOAD_DETAILS) + @Param(description = "download progress of a snapshot", since = "4.19.0") + private Map downloadDetails; + public SnapshotResponse() { tags = new LinkedHashSet(); } @@ -190,7 +219,11 @@ public void setState(Snapshot.State state) { this.state = state; } - public void setPhysicaSize(long physicalSize) { + public void setStatus(String status) { + this.status = status; + } + + public void setPhysicalSize(long physicalSize) { this.physicalSize = physicalSize; } @@ -208,6 +241,10 @@ public void setZoneId(String zoneId) { this.zoneId = zoneId; } + public void setZoneName(String zoneName) { + this.zoneName = zoneName; + } + public void setTags(Set tags) { this.tags = tags; } @@ -231,4 +268,24 @@ public void setOsDisplayName(String osDisplayName) { public void setVirtualSize(long virtualSize) { this.virtualSize = virtualSize; } + + public void setDatastoreId(String datastoreId) { + this.datastoreId = datastoreId; + } + + public void setDatastoreName(String datastoreName) { + this.datastoreName = datastoreName; + } + + public void setDatastoreState(String datastoreState) { + this.datastoreState = datastoreState; + } + + public void setDatastoreType(String datastoreType) { + this.datastoreType = datastoreType; + } + + public void setDownloadDetails(Map downloadDetails) { + this.downloadDetails = downloadDetails; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/ZoneResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/ZoneResponse.java index b8824fd66f89..4e8e665836c3 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/ZoneResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/ZoneResponse.java @@ -95,7 +95,7 @@ public class ZoneResponse extends BaseResponseWithAnnotations implements SetReso @SerializedName("securitygroupsenabled") @Param(description = "true if security groups support is enabled, false otherwise") - private boolean securityGroupsEnabled; + private Boolean securityGroupsEnabled; @SerializedName("allocationstate") @Param(description = "the allocation state of the cluster") @@ -115,7 +115,7 @@ public class ZoneResponse extends BaseResponseWithAnnotations implements SetReso @SerializedName(ApiConstants.LOCAL_STORAGE_ENABLED) @Param(description = "true if local storage offering enabled, false otherwise") - private boolean localStorageEnabled; + private Boolean localStorageEnabled; @SerializedName(ApiConstants.TAGS) @Param(description = "the list of resource tags associated with zone.", responseObject = ResourceTagResponse.class, since = "4.3") @@ -131,7 +131,7 @@ public class ZoneResponse extends BaseResponseWithAnnotations implements SetReso @SerializedName(ApiConstants.ALLOW_USER_SPECIFY_VR_MTU) @Param(description = "Allow end users to specify VR MTU", since = "4.18.0") - private boolean allowUserSpecifyVRMtu; + private Boolean allowUserSpecifyVRMtu; @SerializedName(ApiConstants.ROUTER_PRIVATE_INTERFACE_MAX_MTU) @Param(description = "The maximum value the MTU can have on the VR's private interfaces", since = "4.18.0") @@ -197,7 +197,7 @@ public void setNetworkType(String networkType) { this.networkType = networkType; } - public void setSecurityGroupsEnabled(boolean securityGroupsEnabled) { + public void setSecurityGroupsEnabled(Boolean securityGroupsEnabled) { this.securityGroupsEnabled = securityGroupsEnabled; } @@ -221,7 +221,7 @@ public void setDomainName(String domainName) { this.domainName = domainName; } - public void setLocalStorageEnabled(boolean localStorageEnabled) { + public void setLocalStorageEnabled(Boolean localStorageEnabled) { this.localStorageEnabled = localStorageEnabled; } @@ -241,6 +241,10 @@ public void setIp6Dns2(String ip6Dns2) { this.ip6Dns2 = ip6Dns2; } + public void setTags(Set tags) { + this.tags = tags; + } + public void addTag(ResourceTagResponse tag) { this.tags.add(tag); } @@ -345,7 +349,7 @@ public ResourceIconResponse getResourceIconResponse() { return resourceIconResponse; } - public void setAllowUserSpecifyVRMtu(boolean allowUserSpecifyVRMtu) { + public void setAllowUserSpecifyVRMtu(Boolean allowUserSpecifyVRMtu) { this.allowUserSpecifyVRMtu = allowUserSpecifyVRMtu; } diff --git a/api/src/main/java/org/apache/cloudstack/query/QueryService.java b/api/src/main/java/org/apache/cloudstack/query/QueryService.java index 0587294a826f..097a3c3f2620 100644 --- a/api/src/main/java/org/apache/cloudstack/query/QueryService.java +++ b/api/src/main/java/org/apache/cloudstack/query/QueryService.java @@ -44,6 +44,8 @@ import org.apache.cloudstack.api.command.user.project.ListProjectsCmd; import org.apache.cloudstack.api.command.user.resource.ListDetailOptionsCmd; import org.apache.cloudstack.api.command.user.securitygroup.ListSecurityGroupsCmd; +import org.apache.cloudstack.api.command.user.snapshot.CopySnapshotCmd; +import org.apache.cloudstack.api.command.user.snapshot.ListSnapshotsCmd; import org.apache.cloudstack.api.command.user.tag.ListTagsCmd; import org.apache.cloudstack.api.command.user.template.ListTemplatesCmd; import org.apache.cloudstack.api.command.user.vm.ListVMsCmd; @@ -73,6 +75,7 @@ import org.apache.cloudstack.api.response.RouterHealthCheckResultResponse; import org.apache.cloudstack.api.response.SecurityGroupResponse; import org.apache.cloudstack.api.response.ServiceOfferingResponse; +import org.apache.cloudstack.api.response.SnapshotResponse; import org.apache.cloudstack.api.response.StoragePoolResponse; import org.apache.cloudstack.api.response.StorageTagResponse; import org.apache.cloudstack.api.response.TemplateResponse; @@ -179,4 +182,8 @@ public interface QueryService { ListResponse listManagementServers(ListMgmtsCmd cmd); List listRouterHealthChecks(GetRouterHealthCheckResultsCmd cmd); + + ListResponse listSnapshots(ListSnapshotsCmd cmd); + + SnapshotResponse listSnapshot(CopySnapshotCmd cmd); } diff --git a/api/src/test/java/org/apache/cloudstack/api/command/test/CreateSnapshotCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/test/CreateSnapshotCmdTest.java index 0d3251a64dff..c5288067e943 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/test/CreateSnapshotCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/test/CreateSnapshotCmdTest.java @@ -23,6 +23,7 @@ import static org.mockito.Matchers.isNull; import java.util.HashMap; +import java.util.List; import java.util.Map; import org.apache.cloudstack.api.ResponseGenerator; @@ -92,7 +93,7 @@ public void testCreateSuccess() { Snapshot snapshot = Mockito.mock(Snapshot.class); try { Mockito.when(volumeApiService.takeSnapshot(nullable(Long.class), nullable(Long.class), isNull(), - nullable(Account.class), nullable(Boolean.class), nullable(Snapshot.LocationType.class), nullable(Boolean.class), nullable(Map.class))).thenReturn(snapshot); + nullable(Account.class), nullable(Boolean.class), nullable(Snapshot.LocationType.class), nullable(Boolean.class), nullable(Map.class), nullable(List.class))).thenReturn(snapshot); } catch (Exception e) { Assert.fail("Received exception when success expected " + e.getMessage()); @@ -125,7 +126,7 @@ public void testCreateFailure() { try { Mockito.when(volumeApiService.takeSnapshot(nullable(Long.class), nullable(Long.class), nullable(Long.class), - nullable(Account.class), nullable(Boolean.class), nullable(Snapshot.LocationType.class), nullable(Boolean.class), anyObject())).thenReturn(null); + nullable(Account.class), nullable(Boolean.class), nullable(Snapshot.LocationType.class), nullable(Boolean.class), anyObject(), Mockito.anyList())).thenReturn(null); } catch (Exception e) { Assert.fail("Received exception when success expected " + e.getMessage()); } @@ -159,4 +160,14 @@ public void testParsingTags() { ReflectionTestUtils.setField(createSnapshotCmd, "tags", tagsParams); Assert.assertEquals(createSnapshotCmd.getTags(), expectedTags); } + + @Test + public void testGetZoneIds() { + final CreateSnapshotCmd cmd = new CreateSnapshotCmd(); + List ids = List.of(400L, 500L); + ReflectionTestUtils.setField(cmd, "zoneIds", ids); + Assert.assertEquals(ids.size(), cmd.getZoneIds().size()); + Assert.assertEquals(ids.get(0), cmd.getZoneIds().get(0)); + Assert.assertEquals(ids.get(1), cmd.getZoneIds().get(1)); + } } diff --git a/api/src/test/java/org/apache/cloudstack/api/command/user/snapshot/CopySnapshotCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/user/snapshot/CopySnapshotCmdTest.java new file mode 100644 index 000000000000..632496ad2151 --- /dev/null +++ b/api/src/test/java/org/apache/cloudstack/api/command/user/snapshot/CopySnapshotCmdTest.java @@ -0,0 +1,133 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.snapshot; + +import java.util.List; +import java.util.UUID; + +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.SnapshotResponse; +import org.apache.cloudstack.query.QueryService; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; +import org.springframework.test.util.ReflectionTestUtils; + +import com.cloud.dc.DataCenter; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.storage.Snapshot; +import com.cloud.storage.snapshot.SnapshotApiService; +import com.cloud.utils.db.UUIDManager; + +@RunWith(MockitoJUnitRunner.class) +public class CopySnapshotCmdTest { + + @Test + public void testGetId() { + final CopySnapshotCmd cmd = new CopySnapshotCmd(); + Long id = 100L; + ReflectionTestUtils.setField(cmd, "id", id); + Assert.assertEquals(id, cmd.getId()); + } + + @Test + public void testGetSourceZoneId() { + final CopySnapshotCmd cmd = new CopySnapshotCmd(); + Long id = 200L; + ReflectionTestUtils.setField(cmd, "sourceZoneId", id); + Assert.assertEquals(id, cmd.getSourceZoneId()); + } + + @Test + public void testGetDestZoneIdWithSingleId() { + final CopySnapshotCmd cmd = new CopySnapshotCmd(); + Long id = 300L; + ReflectionTestUtils.setField(cmd, "destZoneId", id); + Assert.assertEquals(1, cmd.getDestinationZoneIds().size()); + Assert.assertEquals(id, cmd.getDestinationZoneIds().get(0)); + } + + @Test + public void testGetDestZoneIdWithMultipleId() { + final CopySnapshotCmd cmd = new CopySnapshotCmd(); + List ids = List.of(400L, 500L); + ReflectionTestUtils.setField(cmd, "destZoneIds", ids); + Assert.assertEquals(ids.size(), cmd.getDestinationZoneIds().size()); + Assert.assertEquals(ids.get(0), cmd.getDestinationZoneIds().get(0)); + Assert.assertEquals(ids.get(1), cmd.getDestinationZoneIds().get(1)); + } + + @Test + public void testGetDestZoneIdWithBothParams() { + final CopySnapshotCmd cmd = new CopySnapshotCmd(); + List ids = List.of(400L, 500L); + ReflectionTestUtils.setField(cmd, "destZoneIds", ids); + ReflectionTestUtils.setField(cmd, "destZoneId", 100L); + Assert.assertEquals(ids.size(), cmd.getDestinationZoneIds().size()); + Assert.assertEquals(ids.get(0), cmd.getDestinationZoneIds().get(0)); + Assert.assertEquals(ids.get(1), cmd.getDestinationZoneIds().get(1)); + } + + @Test (expected = ServerApiException.class) + public void testExecuteWrongNoParams() { + final CopySnapshotCmd cmd = new CopySnapshotCmd(); + try { + cmd.execute(); + } catch (ResourceUnavailableException e) { + Assert.fail(String.format("Exception: %s", e.getMessage())); + } + } + + @Test (expected = ServerApiException.class) + public void testExecuteWrongBothParams() { + final CopySnapshotCmd cmd = new CopySnapshotCmd(); + List ids = List.of(400L, 500L); + ReflectionTestUtils.setField(cmd, "destZoneIds", ids); + ReflectionTestUtils.setField(cmd, "destZoneId", 100L); + try { + cmd.execute(); + } catch (ResourceUnavailableException e) { + Assert.fail(String.format("Exception: %s", e.getMessage())); + } + } + + @Test + public void testExecuteSuccess() { + SnapshotApiService snapshotApiService = Mockito.mock(SnapshotApiService.class); + QueryService queryService = Mockito.mock(QueryService.class); + UUIDManager uuidManager = Mockito.mock(UUIDManager.class); + final CopySnapshotCmd cmd = new CopySnapshotCmd(); + cmd._snapshotService = snapshotApiService; + cmd._queryService = queryService; + cmd._uuidMgr = uuidManager; + Snapshot snapshot = Mockito.mock(Snapshot.class); + final Long id = 100L; + ReflectionTestUtils.setField(cmd, "destZoneId", id); + SnapshotResponse snapshotResponse = Mockito.mock(SnapshotResponse.class); + try { + Mockito.when(snapshotApiService.copySnapshot(cmd)).thenReturn(snapshot); + Mockito.when(queryService.listSnapshot(cmd)).thenReturn(snapshotResponse); + Mockito.when(uuidManager.getUuid(DataCenter.class, id)).thenReturn(UUID.randomUUID().toString()); + cmd.execute(); + } catch (ResourceAllocationException | ResourceUnavailableException e) { + Assert.fail(String.format("Exception: %s", e.getMessage())); + } + } +} diff --git a/api/src/test/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotPolicyCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotPolicyCmdTest.java index 111ac7081e16..258f29e8ad6c 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotPolicyCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotPolicyCmdTest.java @@ -17,6 +17,7 @@ package org.apache.cloudstack.api.command.user.snapshot; import java.util.HashMap; +import java.util.List; import java.util.Map; import org.junit.Assert; @@ -43,4 +44,14 @@ public void testParsingTags() { ReflectionTestUtils.setField(createSnapshotPolicyCmd, "tags", tagsParams); Assert.assertEquals(createSnapshotPolicyCmd.getTags(), expectedTags); } + + @Test + public void testGetZoneIds() { + final CreateSnapshotPolicyCmd cmd = new CreateSnapshotPolicyCmd(); + List ids = List.of(400L, 500L); + ReflectionTestUtils.setField(cmd, "zoneIds", ids); + Assert.assertEquals(ids.size(), cmd.getZoneIds().size()); + Assert.assertEquals(ids.get(0), cmd.getZoneIds().get(0)); + Assert.assertEquals(ids.get(1), cmd.getZoneIds().get(1)); + } } diff --git a/api/src/test/java/org/apache/cloudstack/api/command/user/snapshot/DeleteSnapshotCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/user/snapshot/DeleteSnapshotCmdTest.java new file mode 100644 index 000000000000..48c4279a1da4 --- /dev/null +++ b/api/src/test/java/org/apache/cloudstack/api/command/user/snapshot/DeleteSnapshotCmdTest.java @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.snapshot; + +import org.junit.Assert; +import org.junit.Test; +import org.springframework.test.util.ReflectionTestUtils; + +public class DeleteSnapshotCmdTest { + + @Test + public void testGetZoneId() { + final DeleteSnapshotCmd cmd = new DeleteSnapshotCmd(); + Long id = 400L; + ReflectionTestUtils.setField(cmd, "zoneId", id); + Assert.assertEquals(id, cmd.getZoneId()); + } +} diff --git a/api/src/test/java/org/apache/cloudstack/api/command/user/snapshot/ListSnapshotsCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/user/snapshot/ListSnapshotsCmdTest.java new file mode 100644 index 000000000000..c882691cb431 --- /dev/null +++ b/api/src/test/java/org/apache/cloudstack/api/command/user/snapshot/ListSnapshotsCmdTest.java @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.snapshot; + +import org.junit.Assert; +import org.junit.Test; +import org.springframework.test.util.ReflectionTestUtils; + +public class ListSnapshotsCmdTest { + + @Test + public void testIsShowUniqueNoValue() { + final ListSnapshotsCmd cmd = new ListSnapshotsCmd(); + Assert.assertTrue(cmd.isShowUnique()); + } + + @Test + public void testIsShowUniqueFalse() { + final ListSnapshotsCmd cmd = new ListSnapshotsCmd(); + ReflectionTestUtils.setField(cmd, "showUnique", false); + Assert.assertFalse(cmd.isShowUnique()); + } + + @Test + public void testIsShowUniqueTrue() { + final ListSnapshotsCmd cmd = new ListSnapshotsCmd(); + ReflectionTestUtils.setField(cmd, "showUnique", true); + Assert.assertTrue(cmd.isShowUnique()); + } + + @Test + public void testGetLocationTypeNoUnique() { + final ListSnapshotsCmd cmd = new ListSnapshotsCmd(); + ReflectionTestUtils.setField(cmd, "locationType", "primary"); + Assert.assertNull(cmd.getLocationType()); + } + + @Test + public void testGetLocationTypeUnique() { + final ListSnapshotsCmd cmd = new ListSnapshotsCmd(); + ReflectionTestUtils.setField(cmd, "showUnique", false); + String value = "secondary"; + ReflectionTestUtils.setField(cmd, "locationType", value); + Assert.assertEquals(value, cmd.getLocationType()); + } +} diff --git a/api/src/test/java/org/apache/cloudstack/api/command/user/template/CreateTemplateCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/user/template/CreateTemplateCmdTest.java new file mode 100644 index 000000000000..d8af670a7b21 --- /dev/null +++ b/api/src/test/java/org/apache/cloudstack/api/command/user/template/CreateTemplateCmdTest.java @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.template; + +import org.junit.Assert; +import org.junit.Test; +import org.springframework.test.util.ReflectionTestUtils; + +public class CreateTemplateCmdTest { + + @Test + public void testGetZoneId() { + final CreateTemplateCmd cmd = new CreateTemplateCmd(); + Long id = 400L; + ReflectionTestUtils.setField(cmd, "zoneId", id); + Assert.assertEquals(id, cmd.getZoneId()); + } +} diff --git a/core/src/main/java/com/cloud/agent/transport/Request.java b/core/src/main/java/com/cloud/agent/transport/Request.java index 28809341f781..241ccd4bbd8b 100644 --- a/core/src/main/java/com/cloud/agent/transport/Request.java +++ b/core/src/main/java/com/cloud/agent/transport/Request.java @@ -32,12 +32,20 @@ import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; -import com.cloud.utils.HumanReadableJson; - import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Level; import org.apache.log4j.Logger; +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.BadCommand; +import com.cloud.agent.api.Command; +import com.cloud.agent.api.SecStorageFirewallCfgCommand.PortConfig; +import com.cloud.exception.UnsupportedVersionException; +import com.cloud.serializer.GsonHelper; +import com.cloud.utils.HumanReadableJson; +import com.cloud.utils.NumbersUtil; +import com.cloud.utils.Pair; +import com.cloud.utils.exception.CloudRuntimeException; import com.google.gson.Gson; import com.google.gson.JsonArray; import com.google.gson.JsonDeserializationContext; @@ -49,16 +57,6 @@ import com.google.gson.JsonSerializer; import com.google.gson.stream.JsonReader; -import com.cloud.agent.api.Answer; -import com.cloud.agent.api.BadCommand; -import com.cloud.agent.api.Command; -import com.cloud.agent.api.SecStorageFirewallCfgCommand.PortConfig; -import com.cloud.exception.UnsupportedVersionException; -import com.cloud.serializer.GsonHelper; -import com.cloud.utils.NumbersUtil; -import com.cloud.utils.Pair; -import com.cloud.utils.exception.CloudRuntimeException; - /** * Request is a simple wrapper around command and answer to add sequencing, * versioning, and flags. Note that the version here represents the changes @@ -253,6 +251,7 @@ public Command[] getCommands() { jsonReader.setLenient(true); _cmds = s_gson.fromJson(jsonReader, (Type)Command[].class); } catch (JsonParseException e) { + s_logger.error("Caught problem while parsing JSON command " + _content, e); _cmds = new Command[] { new BadCommand() }; } catch (RuntimeException e) { s_logger.error("Caught problem with " + _content, e); diff --git a/core/src/main/java/com/cloud/storage/resource/StorageSubsystemCommandHandlerBase.java b/core/src/main/java/com/cloud/storage/resource/StorageSubsystemCommandHandlerBase.java index 4a9a24a9f53d..75d5f49d4c6c 100644 --- a/core/src/main/java/com/cloud/storage/resource/StorageSubsystemCommandHandlerBase.java +++ b/core/src/main/java/com/cloud/storage/resource/StorageSubsystemCommandHandlerBase.java @@ -19,23 +19,23 @@ package com.cloud.storage.resource; -import com.cloud.serializer.GsonHelper; import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand; -import org.apache.cloudstack.storage.to.VolumeObjectTO; -import org.apache.cloudstack.storage.command.CheckDataStoreStoragePolicyComplainceCommand; -import org.apache.log4j.Logger; - import org.apache.cloudstack.storage.command.AttachCommand; +import org.apache.cloudstack.storage.command.CheckDataStoreStoragePolicyComplainceCommand; import org.apache.cloudstack.storage.command.CopyCommand; import org.apache.cloudstack.storage.command.CreateObjectAnswer; import org.apache.cloudstack.storage.command.CreateObjectCommand; import org.apache.cloudstack.storage.command.DeleteCommand; import org.apache.cloudstack.storage.command.DettachCommand; import org.apache.cloudstack.storage.command.IntroduceObjectCmd; +import org.apache.cloudstack.storage.command.QuerySnapshotZoneCopyAnswer; +import org.apache.cloudstack.storage.command.QuerySnapshotZoneCopyCommand; import org.apache.cloudstack.storage.command.ResignatureCommand; import org.apache.cloudstack.storage.command.SnapshotAndCopyCommand; import org.apache.cloudstack.storage.command.StorageSubSystemCommand; import org.apache.cloudstack.storage.command.SyncVolumePathCommand; +import org.apache.cloudstack.storage.to.VolumeObjectTO; +import org.apache.log4j.Logger; import com.cloud.agent.api.Answer; import com.cloud.agent.api.Command; @@ -43,6 +43,7 @@ import com.cloud.agent.api.to.DataStoreTO; import com.cloud.agent.api.to.DataTO; import com.cloud.agent.api.to.DiskTO; +import com.cloud.serializer.GsonHelper; import com.cloud.storage.DataStoreRole; import com.cloud.storage.Volume; import com.google.gson.Gson; @@ -81,6 +82,8 @@ public Answer handleStorageCommands(StorageSubSystemCommand command) { return processor.checkDataStoreStoragePolicyCompliance((CheckDataStoreStoragePolicyComplainceCommand) command); } else if (command instanceof SyncVolumePathCommand) { return processor.syncVolumePath((SyncVolumePathCommand) command); + } else if (command instanceof QuerySnapshotZoneCopyCommand) { + return execute((QuerySnapshotZoneCopyCommand)command); } return new Answer((Command)command, false, "not implemented yet"); @@ -175,6 +178,10 @@ protected Answer execute(DettachCommand cmd) { } } + protected Answer execute(QuerySnapshotZoneCopyCommand cmd) { + return new QuerySnapshotZoneCopyAnswer(cmd, "Unsupported command"); + } + private void logCommand(Command cmd) { try { s_logger.debug(String.format("Executing command %s: [%s].", cmd.getClass().getSimpleName(), s_gogger.toJson(cmd))); diff --git a/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java b/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java index 2e331cab227d..d55c387d820a 100755 --- a/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java +++ b/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java @@ -19,6 +19,8 @@ package com.cloud.storage.template; +import static com.cloud.utils.NumbersUtil.toHumanReadableSize; + import java.io.File; import java.io.IOException; import java.io.InputStream; @@ -27,7 +29,8 @@ import java.net.URISyntaxException; import java.util.Date; -import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.cloudstack.managed.context.ManagedContextRunnable; +import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType; import org.apache.cloudstack.utils.imagestore.ImageStoreUtil; import org.apache.commons.httpclient.Credentials; import org.apache.commons.httpclient.Header; @@ -44,16 +47,12 @@ import org.apache.commons.httpclient.params.HttpMethodParams; import org.apache.log4j.Logger; -import org.apache.cloudstack.managed.context.ManagedContextRunnable; -import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType; - import com.cloud.storage.StorageLayer; import com.cloud.utils.Pair; import com.cloud.utils.UriUtils; +import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.net.Proxy; -import static com.cloud.utils.NumbersUtil.toHumanReadableSize; - /** * Download a template file using HTTP * @@ -247,7 +246,9 @@ private boolean copyBytes(File file, InputStream in, RandomAccessFile out) throw while (!done && status != Status.ABORTED && offset <= remoteSize) { if ((bytes = in.read(block, 0, CHUNK_SIZE)) > -1) { offset = writeBlock(bytes, out, block, offset); - if (!verifyFormat.isVerifiedFormat() && (offset >= 1048576 || offset >= remoteSize)) { //let's check format after we get 1MB or full file + if (!ResourceType.SNAPSHOT.equals(resourceType) && + !verifyFormat.isVerifiedFormat() && + (offset >= 1048576 || offset >= remoteSize)) { //let's check format after we get 1MB or full file verifyFormat.invoke(); } } else { diff --git a/core/src/main/java/com/cloud/storage/template/SimpleHttpMultiFileDownloader.java b/core/src/main/java/com/cloud/storage/template/SimpleHttpMultiFileDownloader.java new file mode 100644 index 000000000000..7a0ce47ec996 --- /dev/null +++ b/core/src/main/java/com/cloud/storage/template/SimpleHttpMultiFileDownloader.java @@ -0,0 +1,481 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.storage.template; + +import static com.cloud.utils.NumbersUtil.toHumanReadableSize; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.RandomAccessFile; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; + +import org.apache.cloudstack.managed.context.ManagedContextRunnable; +import org.apache.cloudstack.storage.command.DownloadCommand; +import org.apache.commons.httpclient.Header; +import org.apache.commons.httpclient.HttpClient; +import org.apache.commons.httpclient.HttpException; +import org.apache.commons.httpclient.HttpMethod; +import org.apache.commons.httpclient.HttpMethodRetryHandler; +import org.apache.commons.httpclient.HttpStatus; +import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager; +import org.apache.commons.httpclient.NoHttpResponseException; +import org.apache.commons.httpclient.methods.GetMethod; +import org.apache.commons.httpclient.methods.HeadMethod; +import org.apache.commons.httpclient.params.HttpMethodParams; +import org.apache.commons.lang3.StringUtils; +import org.apache.log4j.Logger; + +import com.cloud.storage.StorageLayer; + +public class SimpleHttpMultiFileDownloader extends ManagedContextRunnable implements TemplateDownloader { + public static final Logger s_logger = Logger.getLogger(SimpleHttpMultiFileDownloader.class.getName()); + private static final MultiThreadedHttpConnectionManager s_httpClientManager = new MultiThreadedHttpConnectionManager(); + + private static final int CHUNK_SIZE = 1024 * 1024; //1M + private String[] downloadUrls; + private String currentToFile; + public TemplateDownloader.Status currentStatus; + public TemplateDownloader.Status status; + private String errorString = null; + private long totalRemoteSize = 0; + private long currentRemoteSize = 0; + public long downloadTime = 0; + public long currentTotalBytes; + public long totalBytes = 0; + private final HttpClient client; + private GetMethod request; + private boolean resume = false; + private DownloadCompleteCallback completionCallback; + StorageLayer _storage; + boolean inited = true; + + private String toDir; + private long maxTemplateSizeInBytes; + private DownloadCommand.ResourceType resourceType = DownloadCommand.ResourceType.TEMPLATE; + private final HttpMethodRetryHandler retryHandler; + + private HashMap urlFileMap; + + public SimpleHttpMultiFileDownloader(StorageLayer storageLayer, String[] downloadUrls, String toDir, + DownloadCompleteCallback callback, long maxTemplateSizeInBytes, + DownloadCommand.ResourceType resourceType) { + _storage = storageLayer; + this.downloadUrls = downloadUrls; + this.toDir = toDir; + this.resourceType = resourceType; + this.maxTemplateSizeInBytes = maxTemplateSizeInBytes; + completionCallback = callback; + status = TemplateDownloader.Status.NOT_STARTED; + currentStatus = TemplateDownloader.Status.NOT_STARTED; + currentTotalBytes = 0; + client = new HttpClient(s_httpClientManager); + retryHandler = createRetryTwiceHandler(); + urlFileMap = new HashMap<>(); + } + + private GetMethod createRequest(String downloadUrl) { + GetMethod request = new GetMethod(downloadUrl); + request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, retryHandler); + request.setFollowRedirects(true); + return request; + } + + private void checkTemporaryDestination(String toDir) { + try { + File f = File.createTempFile("dnld", "tmp_", new File(toDir)); + if (_storage != null) { + _storage.setWorldReadableAndWriteable(f); + } + currentToFile = f.getAbsolutePath(); + } catch (IOException ex) { + errorString = "Unable to start download -- check url? "; + currentStatus = TemplateDownloader.Status.UNRECOVERABLE_ERROR; + s_logger.warn("Exception in constructor -- " + ex.toString()); + } + } + + private HttpMethodRetryHandler createRetryTwiceHandler() { + return new HttpMethodRetryHandler() { + @Override + public boolean retryMethod(final HttpMethod method, final IOException exception, int executionCount) { + if (executionCount >= 2) { + // Do not retry if over max retry count + return false; + } + if (exception instanceof NoHttpResponseException) { + // Retry if the server dropped connection on us + return true; + } + if (!method.isRequestSent()) { + // Retry if the request has not been sent fully or + // if it's OK to retry methods that have been sent + return true; + } + // otherwise do not retry + return false; + } + }; + } + + private void tryAndGetTotalRemoteSize() { + for (String downloadUrl : downloadUrls) { + if (StringUtils.isBlank(downloadUrl)) { + continue; + } + HeadMethod headMethod = new HeadMethod(downloadUrl); + try { + if (client.executeMethod(headMethod) != HttpStatus.SC_OK) { + continue; + } + Header contentLengthHeader = headMethod.getResponseHeader("content-length"); + if (contentLengthHeader == null) { + continue; + } + totalRemoteSize += Long.parseLong(contentLengthHeader.getValue()); + } catch (IOException e) { + s_logger.warn(String.format("Cannot reach URL: %s while trying to get remote sizes due to: %s", downloadUrl, e.getMessage()), e); + } finally { + headMethod.releaseConnection(); + } + } + } + + private long downloadFile(String downloadUrl) { + s_logger.debug("Starting download for " + downloadUrl); + currentTotalBytes = 0; + currentRemoteSize = 0; + File file = null; + request = null; + try { + request = createRequest(downloadUrl); + checkTemporaryDestination(toDir); + urlFileMap.put(downloadUrl, currentToFile); + file = new File(currentToFile); + long localFileSize = checkLocalFileSizeForResume(resume, file); + if (checkServerResponse(localFileSize)) return 0; + if (!tryAndGetRemoteSize()) return 0; + if (!canHandleDownloadSize()) return 0; + checkAndSetDownloadSize(); + try (InputStream in = request.getResponseBodyAsStream(); + RandomAccessFile out = new RandomAccessFile(file, "rw"); + ) { + out.seek(localFileSize); + s_logger.info("Starting download from " + downloadUrl + " to " + currentToFile + " remoteSize=" + toHumanReadableSize(currentRemoteSize) + " , max size=" + toHumanReadableSize(maxTemplateSizeInBytes)); + if (copyBytes(file, in, out)) return 0; + checkDownloadCompletion(); + } + return currentTotalBytes; + } catch (HttpException hte) { + currentStatus = TemplateDownloader.Status.UNRECOVERABLE_ERROR; + errorString = hte.getMessage(); + } catch (IOException ioe) { + currentStatus = TemplateDownloader.Status.UNRECOVERABLE_ERROR; //probably a file write error? + // Let's not overwrite the original error message. + if (errorString == null) { + errorString = ioe.getMessage(); + } + } finally { + if (currentStatus == Status.UNRECOVERABLE_ERROR && file != null && file.exists() && !file.isDirectory()) { + file.delete(); + } + if (request != null) { + request.releaseConnection(); + } + } + return 0; + } + + @Override + public long download(boolean resume, DownloadCompleteCallback callback) { + if (skipDownloadOnStatus()) return 0; + if (resume) { + s_logger.error("Resume not allowed for this downloader"); + status = Status.UNRECOVERABLE_ERROR; + return 0; + } + s_logger.debug("Starting downloads"); + status = Status.IN_PROGRESS; + Date start = new Date(); + tryAndGetTotalRemoteSize(); + for (String downloadUrl : downloadUrls) { + if (StringUtils.isBlank(downloadUrl)) { + continue; + } + long bytes = downloadFile(downloadUrl); + if (currentStatus != Status.DOWNLOAD_FINISHED) { + break; + } + totalBytes += bytes; + } + status = currentStatus; + Date finish = new Date(); + downloadTime += finish.getTime() - start.getTime(); + if (callback != null) { + callback.downloadComplete(status); + } + return 0; + } + + private boolean copyBytes(File file, InputStream in, RandomAccessFile out) throws IOException { + int bytes; + byte[] block = new byte[CHUNK_SIZE]; + long offset = 0; + boolean done = false; + currentStatus = Status.IN_PROGRESS; + while (!done && currentStatus != Status.ABORTED && offset <= currentRemoteSize) { + if ((bytes = in.read(block, 0, CHUNK_SIZE)) > -1) { + offset = writeBlock(bytes, out, block, offset); + } else { + done = true; + } + } + out.getFD().sync(); + return false; + } + + private long writeBlock(int bytes, RandomAccessFile out, byte[] block, long offset) throws IOException { + out.write(block, 0, bytes); + offset += bytes; + out.seek(offset); + currentTotalBytes += bytes; + return offset; + } + + private void checkDownloadCompletion() { + String downloaded = "(incomplete download)"; + if (currentTotalBytes >= currentRemoteSize) { + currentStatus = Status.DOWNLOAD_FINISHED; + downloaded = "(download complete remote=" + toHumanReadableSize(currentRemoteSize) + " bytes)"; + } + errorString = "Downloaded " + toHumanReadableSize(currentTotalBytes) + " bytes " + downloaded; + } + + private boolean canHandleDownloadSize() { + if (currentRemoteSize > maxTemplateSizeInBytes) { + s_logger.info("Remote size is too large: " + toHumanReadableSize(currentRemoteSize) + " , max=" + toHumanReadableSize(maxTemplateSizeInBytes)); + currentStatus = Status.UNRECOVERABLE_ERROR; + errorString = "Download file size is too large"; + return false; + } + return true; + } + + private void checkAndSetDownloadSize() { + if (currentRemoteSize == 0) { + currentRemoteSize = maxTemplateSizeInBytes; + } + if (totalRemoteSize == 0) { + totalRemoteSize = currentRemoteSize; + } + } + + private boolean tryAndGetRemoteSize() { + Header contentLengthHeader = request.getResponseHeader("content-length"); + boolean chunked = false; + long reportedRemoteSize = 0; + if (contentLengthHeader == null) { + Header chunkedHeader = request.getResponseHeader("Transfer-Encoding"); + if (chunkedHeader == null || !"chunked".equalsIgnoreCase(chunkedHeader.getValue())) { + currentStatus = Status.UNRECOVERABLE_ERROR; + errorString = " Failed to receive length of download "; + return false; + } else if ("chunked".equalsIgnoreCase(chunkedHeader.getValue())) { + chunked = true; + } + } else { + reportedRemoteSize = Long.parseLong(contentLengthHeader.getValue()); + if (reportedRemoteSize == 0) { + currentStatus = Status.DOWNLOAD_FINISHED; + String downloaded = "(download complete remote=" + currentRemoteSize + "bytes)"; + errorString = "Downloaded " + currentTotalBytes + " bytes " + downloaded; + downloadTime = 0; + return false; + } + } + + if (currentRemoteSize == 0) { + currentRemoteSize = reportedRemoteSize; + } + return true; + } + + private boolean checkServerResponse(long localFileSize) throws IOException { + int responseCode = 0; + + if (localFileSize > 0) { + // require partial content support for resume + request.addRequestHeader("Range", "bytes=" + localFileSize + "-"); + if (client.executeMethod(request) != HttpStatus.SC_PARTIAL_CONTENT) { + errorString = "HTTP Server does not support partial get"; + currentStatus = Status.UNRECOVERABLE_ERROR; + return true; + } + } else if ((responseCode = client.executeMethod(request)) != HttpStatus.SC_OK) { + currentStatus = Status.UNRECOVERABLE_ERROR; + errorString = " HTTP Server returned " + responseCode + " (expected 200 OK) "; + return true; //FIXME: retry? + } + return false; + } + + private long checkLocalFileSizeForResume(boolean resume, File file) { + // TODO check the status of this downloader as well? + long localFileSize = 0; + if (file.exists() && resume) { + localFileSize = file.length(); + s_logger.info("Resuming download to file (current size)=" + toHumanReadableSize(localFileSize)); + } + return localFileSize; + } + + private boolean skipDownloadOnStatus() { + switch (currentStatus) { + case ABORTED: + case UNRECOVERABLE_ERROR: + case DOWNLOAD_FINISHED: + return true; + default: + + } + return false; + } + + public String[] getDownloadUrls() { + return downloadUrls; + } + + public String getCurrentToFile() { + File file = new File(currentToFile); + + return file.getAbsolutePath(); + } + + @Override + public TemplateDownloader.Status getStatus() { + return currentStatus; + } + + @Override + public long getDownloadTime() { + return downloadTime; + } + + @Override + public long getDownloadedBytes() { + return totalBytes; + } + + @Override + @SuppressWarnings("fallthrough") + public boolean stopDownload() { + switch (getStatus()) { + case IN_PROGRESS: + if (request != null) { + request.abort(); + } + currentStatus = TemplateDownloader.Status.ABORTED; + return true; + case UNKNOWN: + case NOT_STARTED: + case RECOVERABLE_ERROR: + case UNRECOVERABLE_ERROR: + case ABORTED: + currentStatus = TemplateDownloader.Status.ABORTED; + case DOWNLOAD_FINISHED: + File f = new File(currentToFile); + if (f.exists()) { + f.delete(); + } + return true; + + default: + return true; + } + } + + @Override + public int getDownloadPercent() { + if (totalRemoteSize == 0) { + return 0; + } + + return (int)(100.0 * totalBytes / totalRemoteSize); + } + + @Override + protected void runInContext() { + try { + download(resume, completionCallback); + } catch (Throwable t) { + s_logger.warn("Caught exception during download " + t.getMessage(), t); + errorString = "Failed to install: " + t.getMessage(); + currentStatus = TemplateDownloader.Status.UNRECOVERABLE_ERROR; + } + + } + + @Override + public void setStatus(TemplateDownloader.Status status) { + this.currentStatus = status; + } + + public boolean isResume() { + return resume; + } + + @Override + public String getDownloadError() { + return errorString == null ? " " : errorString; + } + + @Override + public String getDownloadLocalPath() { + return toDir; + } + + @Override + public void setResume(boolean resume) { + this.resume = resume; + } + + @Override + public long getMaxTemplateSizeInBytes() { + return maxTemplateSizeInBytes; + } + + @Override + public void setDownloadError(String error) { + errorString = error; + } + + @Override + public boolean isInited() { + return inited; + } + + public DownloadCommand.ResourceType getResourceType() { + return resourceType; + } + + public Map getDownloadedFilesMap() { + return urlFileMap; + } +} diff --git a/core/src/main/java/com/cloud/storage/template/TemplateLocation.java b/core/src/main/java/com/cloud/storage/template/TemplateLocation.java index 99360eea72c2..6ff53a0410a9 100644 --- a/core/src/main/java/com/cloud/storage/template/TemplateLocation.java +++ b/core/src/main/java/com/cloud/storage/template/TemplateLocation.java @@ -19,26 +19,25 @@ package com.cloud.storage.template; +import static com.cloud.utils.NumbersUtil.toHumanReadableSize; + import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Iterator; import java.util.Properties; -import java.util.Arrays; - -import org.apache.log4j.Logger; import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType; +import org.apache.log4j.Logger; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.StorageLayer; import com.cloud.storage.template.Processor.FormatInfo; import com.cloud.utils.NumbersUtil; -import static com.cloud.utils.NumbersUtil.toHumanReadableSize; - public class TemplateLocation { private static final Logger s_logger = Logger.getLogger(TemplateLocation.class); public final static String Filename = "template.properties"; @@ -65,6 +64,9 @@ public TemplateLocation(StorageLayer storage, String templatePath) { if (_templatePath.matches(".*" + "volumes" + ".*")) { _file = _storage.getFile(_templatePath + "volume.properties"); _resourceType = ResourceType.VOLUME; + } else if (_templatePath.matches(".*" + "snapshots" + ".*")) { + _file = _storage.getFile(_templatePath + "snapshot.properties"); + _resourceType = ResourceType.SNAPSHOT; } else { _file = _storage.getFile(_templatePath + Filename); } @@ -170,6 +172,8 @@ public TemplateProp getTemplateInfo() { tmplInfo.installPath = _templatePath + _props.getProperty("filename"); // _templatePath endsWith / if (_resourceType == ResourceType.VOLUME) { tmplInfo.installPath = tmplInfo.installPath.substring(tmplInfo.installPath.indexOf("volumes")); + } else if (_resourceType == ResourceType.SNAPSHOT) { + tmplInfo.installPath = tmplInfo.installPath.substring(tmplInfo.installPath.indexOf("snapshots")); } else { tmplInfo.installPath = tmplInfo.installPath.substring(tmplInfo.installPath.indexOf("template")); } diff --git a/core/src/main/java/org/apache/cloudstack/storage/command/DownloadCommand.java b/core/src/main/java/org/apache/cloudstack/storage/command/DownloadCommand.java index 29d737fcce93..4032ac0b6322 100644 --- a/core/src/main/java/org/apache/cloudstack/storage/command/DownloadCommand.java +++ b/core/src/main/java/org/apache/cloudstack/storage/command/DownloadCommand.java @@ -20,6 +20,7 @@ package org.apache.cloudstack.storage.command; import org.apache.cloudstack.api.InternalIdentity; +import org.apache.cloudstack.storage.to.SnapshotObjectTO; import org.apache.cloudstack.storage.to.TemplateObjectTO; import org.apache.cloudstack.storage.to.VolumeObjectTO; @@ -33,7 +34,7 @@ public class DownloadCommand extends AbstractDownloadCommand implements InternalIdentity { public static enum ResourceType { - VOLUME, TEMPLATE + VOLUME, TEMPLATE, SNAPSHOT } private boolean hvm; @@ -96,6 +97,18 @@ public DownloadCommand(VolumeObjectTO volume, Long maxDownloadSizeInBytes, Strin resourceType = ResourceType.VOLUME; } + public DownloadCommand(SnapshotObjectTO snapshot, Long maxDownloadSizeInBytes, String url) { + super(snapshot.getName(), url, null, snapshot.getAccountId()); + _store = snapshot.getDataStore(); + installPath = snapshot.getPath(); + id = snapshot.getId(); + if (_store instanceof NfsTO) { + setSecUrl(((NfsTO)_store).getUrl()); + } + this.maxDownloadSizeInBytes = maxDownloadSizeInBytes; + this.resourceType = ResourceType.SNAPSHOT; + } + @Override public long getId() { return id; diff --git a/core/src/main/java/org/apache/cloudstack/storage/command/QuerySnapshotZoneCopyAnswer.java b/core/src/main/java/org/apache/cloudstack/storage/command/QuerySnapshotZoneCopyAnswer.java new file mode 100644 index 000000000000..7c96225ce540 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/storage/command/QuerySnapshotZoneCopyAnswer.java @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.storage.command; + +import java.util.List; + +import com.cloud.agent.api.Answer; + +public class QuerySnapshotZoneCopyAnswer extends Answer { + private List files; + + public QuerySnapshotZoneCopyAnswer(QuerySnapshotZoneCopyCommand cmd, List files) { + super(cmd); + this.files = files; + } + + public QuerySnapshotZoneCopyAnswer(QuerySnapshotZoneCopyCommand cmd, String errMsg) { + super(null, false, errMsg); + } + + public List getFiles() { + return files; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/storage/command/QuerySnapshotZoneCopyCommand.java b/core/src/main/java/org/apache/cloudstack/storage/command/QuerySnapshotZoneCopyCommand.java new file mode 100644 index 000000000000..5bca52484ebb --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/storage/command/QuerySnapshotZoneCopyCommand.java @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.storage.command; + +import org.apache.cloudstack.storage.to.SnapshotObjectTO; + +/* +Command to get the list of snapshot files for copying a snapshot to a different zone + */ + +public class QuerySnapshotZoneCopyCommand extends StorageSubSystemCommand { + + private SnapshotObjectTO snapshot; + + public QuerySnapshotZoneCopyCommand(final SnapshotObjectTO snapshot) { + super(); + this.snapshot = snapshot; + } + + public SnapshotObjectTO getSnapshot() { + return snapshot; + } + + public void setSnapshot(final SnapshotObjectTO snapshot) { + this.snapshot = snapshot; + } + + @Override + public boolean executeInSequence() { + return false; + } + + @Override + public void setExecuteInSequence(boolean inSeq) {} +} diff --git a/core/src/main/java/org/apache/cloudstack/storage/to/SnapshotObjectTO.java b/core/src/main/java/org/apache/cloudstack/storage/to/SnapshotObjectTO.java index c62110b179ec..70cb6d155b04 100644 --- a/core/src/main/java/org/apache/cloudstack/storage/to/SnapshotObjectTO.java +++ b/core/src/main/java/org/apache/cloudstack/storage/to/SnapshotObjectTO.java @@ -42,6 +42,7 @@ public class SnapshotObjectTO implements DataTO { private boolean quiescevm; private String[] parents; private Long physicalSize = (long) 0; + private long accountId; public SnapshotObjectTO() { @@ -51,6 +52,7 @@ public SnapshotObjectTO() { public SnapshotObjectTO(SnapshotInfo snapshot) { this.path = snapshot.getPath(); this.setId(snapshot.getId()); + this.accountId = snapshot.getAccountId(); VolumeInfo vol = snapshot.getBaseVolume(); if (vol != null) { this.volume = (VolumeObjectTO)vol.getTO(); @@ -168,6 +170,14 @@ public String[] getParents() { return parents; } + public long getAccountId() { + return accountId; + } + + public void setAccountId(long accountId) { + this.accountId = accountId; + } + @Override public String toString() { return new StringBuilder("SnapshotTO[datastore=").append(dataStore).append("|volume=").append(volume).append("|path").append(path).append("]").toString(); diff --git a/core/src/test/java/com/cloud/storage/resource/StorageSubsystemCommandHandlerBaseTest.java b/core/src/test/java/com/cloud/storage/resource/StorageSubsystemCommandHandlerBaseTest.java new file mode 100644 index 000000000000..104c6521676b --- /dev/null +++ b/core/src/test/java/com/cloud/storage/resource/StorageSubsystemCommandHandlerBaseTest.java @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.storage.resource; + +import org.apache.cloudstack.storage.command.QuerySnapshotZoneCopyAnswer; +import org.apache.cloudstack.storage.command.QuerySnapshotZoneCopyCommand; +import org.apache.cloudstack.storage.to.SnapshotObjectTO; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.agent.api.Answer; + +@RunWith(MockitoJUnitRunner.class) +public class StorageSubsystemCommandHandlerBaseTest { + + @Test + public void testHandleQuerySnapshotCommand() { + StorageSubsystemCommandHandlerBase storageSubsystemCommandHandlerBase = new StorageSubsystemCommandHandlerBase(Mockito.mock(StorageProcessor.class)); + QuerySnapshotZoneCopyCommand querySnapshotZoneCopyCommand = new QuerySnapshotZoneCopyCommand(Mockito.mock(SnapshotObjectTO.class)); + Answer answer = storageSubsystemCommandHandlerBase.handleStorageCommands(querySnapshotZoneCopyCommand); + Assert.assertTrue(answer instanceof QuerySnapshotZoneCopyAnswer); + QuerySnapshotZoneCopyAnswer querySnapshotZoneCopyAnswer = (QuerySnapshotZoneCopyAnswer)answer; + Assert.assertFalse(querySnapshotZoneCopyAnswer.getResult()); + Assert.assertEquals("Unsupported command", querySnapshotZoneCopyAnswer.getDetails()); + } +} diff --git a/core/src/test/java/org/apache/cloudstack/storage/command/DownloadCommandTest.java b/core/src/test/java/org/apache/cloudstack/storage/command/DownloadCommandTest.java new file mode 100644 index 000000000000..59263b1c1873 --- /dev/null +++ b/core/src/test/java/org/apache/cloudstack/storage/command/DownloadCommandTest.java @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.command; + +import org.apache.cloudstack.storage.to.SnapshotObjectTO; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; + +public class DownloadCommandTest { + + @Test + public void testDownloadCOmmandSnapshot() { + SnapshotObjectTO snapshotObjectTO = Mockito.mock(SnapshotObjectTO.class); + Long maxDownloadSizeInBytes = 1000L; + String url = "SOMEURL"; + DownloadCommand cmd = new DownloadCommand(snapshotObjectTO, maxDownloadSizeInBytes, url); + Assert.assertEquals(DownloadCommand.ResourceType.SNAPSHOT, cmd.getResourceType()); + Assert.assertEquals(maxDownloadSizeInBytes, cmd.getMaxDownloadSizeInBytes()); + Assert.assertEquals(url, cmd.getUrl()); + } +} diff --git a/core/src/test/java/org/apache/cloudstack/storage/command/QuerySnapshotZoneCopyAnswerTest.java b/core/src/test/java/org/apache/cloudstack/storage/command/QuerySnapshotZoneCopyAnswerTest.java new file mode 100644 index 000000000000..73221ebad351 --- /dev/null +++ b/core/src/test/java/org/apache/cloudstack/storage/command/QuerySnapshotZoneCopyAnswerTest.java @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.command; + +import java.util.List; + +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; + +public class QuerySnapshotZoneCopyAnswerTest { + + @Test + public void testQuerySnapshotZoneCopyAnswerSuccess() { + QuerySnapshotZoneCopyCommand cmd = Mockito.mock(QuerySnapshotZoneCopyCommand.class); + List files = List.of("File1", "File2"); + QuerySnapshotZoneCopyAnswer answer = new QuerySnapshotZoneCopyAnswer(cmd, files); + Assert.assertTrue(answer.getResult()); + Assert.assertEquals(files.size(), answer.getFiles().size()); + Assert.assertEquals(files.get(0), answer.getFiles().get(0)); + Assert.assertEquals(files.get(1), answer.getFiles().get(1)); + } + + @Test + public void testQuerySnapshotZoneCopyAnswerFailure() { + QuerySnapshotZoneCopyCommand cmd = Mockito.mock(QuerySnapshotZoneCopyCommand.class); + String err = "SOMEERROR"; + QuerySnapshotZoneCopyAnswer answer = new QuerySnapshotZoneCopyAnswer(cmd, err); + Assert.assertFalse(answer.getResult()); + Assert.assertEquals(err, answer.getDetails()); + } +} diff --git a/core/src/test/java/org/apache/cloudstack/storage/to/SnapshotObjectTOTest.java b/core/src/test/java/org/apache/cloudstack/storage/to/SnapshotObjectTOTest.java new file mode 100644 index 000000000000..e76bfb038ca7 --- /dev/null +++ b/core/src/test/java/org/apache/cloudstack/storage/to/SnapshotObjectTOTest.java @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.to; + +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; +import org.springframework.test.util.ReflectionTestUtils; + +public class SnapshotObjectTOTest { + + @Test + public void testAccountId() { + SnapshotObjectTO obj = new SnapshotObjectTO(); + long accountId = 1L; + ReflectionTestUtils.setField(obj, "accountId", accountId); + Assert.assertEquals(accountId, obj.getAccountId()); + accountId = 100L; + obj.setAccountId(accountId); + Assert.assertEquals(accountId, obj.getAccountId()); + SnapshotInfo snapshot = Mockito.mock(SnapshotInfo.class); + Mockito.when(snapshot.getAccountId()).thenReturn(accountId); + Mockito.when(snapshot.getDataStore()).thenReturn(Mockito.mock(DataStore.class)); + SnapshotObjectTO object = new SnapshotObjectTO(snapshot); + Assert.assertEquals(accountId, object.getAccountId()); + } +} diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreManager.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreManager.java index 80e3ce11c759..3ee5803a91a7 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreManager.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreManager.java @@ -54,4 +54,6 @@ public interface DataStoreManager { List listImageCacheStores(); boolean isRegionStore(DataStore store); + + Long getStoreZoneId(long storeId, DataStoreRole role); } diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotDataFactory.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotDataFactory.java index 86f0ab8bcf28..2a2db4af3e53 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotDataFactory.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotDataFactory.java @@ -27,11 +27,17 @@ public interface SnapshotDataFactory { SnapshotInfo getSnapshot(DataObject obj, DataStore store); - SnapshotInfo getSnapshot(long snapshotId, DataStoreRole role); + SnapshotInfo getSnapshot(long snapshotId, long storeId, DataStoreRole role); - SnapshotInfo getSnapshot(long snapshotId, DataStoreRole role, boolean retrieveAnySnapshotFromVolume); + SnapshotInfo getSnapshotWithRoleAndZone(long snapshotId, DataStoreRole role, long zoneId); - List getSnapshots(long volumeId, DataStoreRole store); + SnapshotInfo getSnapshotOnPrimaryStore(long snapshotId); + + SnapshotInfo getSnapshot(long snapshotId, DataStoreRole role, long zoneId, boolean retrieveAnySnapshotFromVolume); + + List getSnapshotsForVolumeAndStoreRole(long volumeId, DataStoreRole store); + + List getSnapshots(long snapshotId, Long zoneId); List listSnapshotOnCache(long snapshotId); diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotInfo.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotInfo.java index ecc412aa79de..3213484694eb 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotInfo.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotInfo.java @@ -57,4 +57,6 @@ public interface SnapshotInfo extends DataObject, Snapshot { void markBackedUp() throws CloudRuntimeException; Snapshot getSnapshotVO(); + + long getAccountId(); } diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotService.java index 053e0cdd1340..d2e085fe90cf 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotService.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotService.java @@ -17,6 +17,9 @@ package org.apache.cloudstack.engine.subsystem.api.storage; +import org.apache.cloudstack.framework.async.AsyncCallFuture; + +import com.cloud.exception.ResourceUnavailableException; import com.cloud.storage.Snapshot.Event; public interface SnapshotService { @@ -35,4 +38,8 @@ public interface SnapshotService { void processEventOnSnapshotObject(SnapshotInfo snapshot, Event event); void cleanupOnSnapshotBackupFailure(SnapshotInfo snapshot); + + AsyncCallFuture copySnapshot(SnapshotInfo snapshot, String copyUrl, DataStore dataStore) throws ResourceUnavailableException; + + AsyncCallFuture queryCopySnapshot(SnapshotInfo snapshot) throws ResourceUnavailableException; } diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotStrategy.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotStrategy.java index 62c4c20ec7b6..f3aa8f52c932 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotStrategy.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotStrategy.java @@ -28,11 +28,11 @@ enum SnapshotOperation { SnapshotInfo backupSnapshot(SnapshotInfo snapshot); - boolean deleteSnapshot(Long snapshotId); + boolean deleteSnapshot(Long snapshotId, Long zoneId); boolean revertSnapshot(SnapshotInfo snapshot); - StrategyPriority canHandle(Snapshot snapshot, SnapshotOperation op); + StrategyPriority canHandle(Snapshot snapshot, Long zoneId, SnapshotOperation op); void postSnapshotCreation(SnapshotInfo snapshot); } diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/StorageStrategyFactory.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/StorageStrategyFactory.java index e309b9842be5..deaee439b3db 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/StorageStrategyFactory.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/StorageStrategyFactory.java @@ -34,6 +34,8 @@ public interface StorageStrategyFactory { SnapshotStrategy getSnapshotStrategy(Snapshot snapshot, SnapshotOperation op); + SnapshotStrategy getSnapshotStrategy(Snapshot snapshot, Long zoneId, SnapshotOperation op); + VMSnapshotStrategy getVmSnapshotStrategy(VMSnapshot vmSnapshot); /** diff --git a/engine/components-api/src/main/java/com/cloud/vm/VmWorkTakeVolumeSnapshot.java b/engine/components-api/src/main/java/com/cloud/vm/VmWorkTakeVolumeSnapshot.java index 6d6264406de5..8474052be201 100644 --- a/engine/components-api/src/main/java/com/cloud/vm/VmWorkTakeVolumeSnapshot.java +++ b/engine/components-api/src/main/java/com/cloud/vm/VmWorkTakeVolumeSnapshot.java @@ -16,6 +16,8 @@ // under the License. package com.cloud.vm; +import java.util.List; + import com.cloud.storage.Snapshot; public class VmWorkTakeVolumeSnapshot extends VmWork { @@ -29,8 +31,11 @@ public class VmWorkTakeVolumeSnapshot extends VmWork { private Snapshot.LocationType locationType; private boolean asyncBackup; + private List zoneIds; + public VmWorkTakeVolumeSnapshot(long userId, long accountId, long vmId, String handlerName, - Long volumeId, Long policyId, Long snapshotId, boolean quiesceVm, Snapshot.LocationType locationType, boolean asyncBackup) { + Long volumeId, Long policyId, Long snapshotId, boolean quiesceVm, Snapshot.LocationType locationType, + boolean asyncBackup, List zoneIds) { super(userId, accountId, vmId, handlerName); this.volumeId = volumeId; this.policyId = policyId; @@ -38,6 +43,7 @@ public VmWorkTakeVolumeSnapshot(long userId, long accountId, long vmId, String h this.quiesceVm = quiesceVm; this.locationType = locationType; this.asyncBackup = asyncBackup; + this.zoneIds = zoneIds; } public Long getVolumeId() { @@ -61,4 +67,8 @@ public boolean isQuiesceVm() { public boolean isAsyncBackup() { return asyncBackup; } + + public List getZoneIds() { + return zoneIds; + } } diff --git a/engine/components-api/src/test/java/com/cloud/vm/VmWorkTakeVolumeSnapshotTest.java b/engine/components-api/src/test/java/com/cloud/vm/VmWorkTakeVolumeSnapshotTest.java new file mode 100644 index 000000000000..feb7ee46aec5 --- /dev/null +++ b/engine/components-api/src/test/java/com/cloud/vm/VmWorkTakeVolumeSnapshotTest.java @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.vm; + +import java.util.List; + +import org.junit.Assert; +import org.junit.Test; + +public class VmWorkTakeVolumeSnapshotTest { + + @Test + public void testVmWorkTakeVolumeSnapshotZoneIds() { + List zoneIds = List.of(10L, 20L); + VmWorkTakeVolumeSnapshot work = new VmWorkTakeVolumeSnapshot(1L, 1L, 1L, "handler", + 1L, 1L, 1L, false, null, false, zoneIds); + Assert.assertNotNull(work.getZoneIds()); + Assert.assertEquals(zoneIds.size(), work.getZoneIds().size()); + Assert.assertEquals(zoneIds.get(0), work.getZoneIds().get(0)); + Assert.assertEquals(zoneIds.get(1), work.getZoneIds().get(1)); + } +} diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/DataMigrationUtility.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/DataMigrationUtility.java index 71b1281decb7..ea6318f05910 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/DataMigrationUtility.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/DataMigrationUtility.java @@ -46,6 +46,7 @@ import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreDao; import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO; +import org.apache.log4j.Logger; import com.cloud.host.HostVO; import com.cloud.host.Status; @@ -61,7 +62,6 @@ import com.cloud.vm.SecondaryStorageVmVO; import com.cloud.vm.VirtualMachine; import com.cloud.vm.dao.SecondaryStorageVmDao; -import org.apache.log4j.Logger; public class DataMigrationUtility { private static Logger LOGGER = Logger.getLogger(DataMigrationUtility.class); @@ -223,7 +223,7 @@ protected List getAllReadySnapshotsAndChains(DataStore srcDataStore, if (snapshot.getState() == ObjectInDataStoreStateMachine.State.Ready && snapshotVO != null && snapshotVO.getHypervisorType() != Hypervisor.HypervisorType.Simulator && snapshot.getParentSnapshotId() == 0 ) { - SnapshotInfo snap = snapshotFactory.getSnapshot(snapshotVO.getSnapshotId(), DataStoreRole.Image); + SnapshotInfo snap = snapshotFactory.getSnapshot(snapshotVO.getSnapshotId(), snapshot.getDataStoreId(), snapshot.getRole()); if (snap != null) { files.add(snap); } diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/StorageOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/StorageOrchestrator.java index 01c7f723ea2f..eef0cdef2fee 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/StorageOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/StorageOrchestrator.java @@ -17,7 +17,6 @@ package org.apache.cloudstack.engine.orchestration; -import com.cloud.capacity.CapacityManager; import java.util.ArrayList; import java.util.Date; import java.util.HashMap; @@ -58,6 +57,7 @@ import org.apache.commons.math3.stat.descriptive.moment.StandardDeviation; import org.apache.log4j.Logger; +import com.cloud.capacity.CapacityManager; import com.cloud.server.StatsCollector; import com.cloud.storage.DataStoreRole; import com.cloud.storage.SnapshotVO; @@ -305,7 +305,7 @@ private void handleSnapshotMigration(Long srcDataStoreId, Date start, Date end, if (!snaps.isEmpty()) { for (SnapshotDataStoreVO snap : snaps) { SnapshotVO snapshotVO = snapshotDao.findById(snap.getSnapshotId()); - SnapshotInfo snapshotInfo = snapshotFactory.getSnapshot(snapshotVO.getSnapshotId(), DataStoreRole.Image); + SnapshotInfo snapshotInfo = snapshotFactory.getSnapshot(snapshotVO.getSnapshotId(), snap.getDataStoreId(), DataStoreRole.Image); SnapshotInfo parentSnapshot = snapshotInfo.getParent(); if (parentSnapshot == null && policy == MigrationPolicy.COMPLETE) { diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java index c3908795f7cb..6f945479bd49 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java @@ -558,7 +558,7 @@ public VolumeInfo createVolumeFromSnapshot(Volume volume, Snapshot snapshot, Use VolumeInfo vol = volFactory.getVolume(volume.getId()); DataStore store = dataStoreMgr.getDataStore(pool.getId(), DataStoreRole.Primary); DataStoreRole dataStoreRole = snapshotHelper.getDataStoreRole(snapshot); - SnapshotInfo snapInfo = snapshotFactory.getSnapshot(snapshot.getId(), dataStoreRole); + SnapshotInfo snapInfo = snapshotFactory.getSnapshotWithRoleAndZone(snapshot.getId(), dataStoreRole, volume.getDataCenterId()); boolean kvmSnapshotOnlyInPrimaryStorage = snapshotHelper.isKvmSnapshotOnlyInPrimaryStorage(snapshot, dataStoreRole); diff --git a/engine/schema/src/main/java/com/cloud/dc/dao/DataCenterDao.java b/engine/schema/src/main/java/com/cloud/dc/dao/DataCenterDao.java index aea51925f9c6..dddbce31772d 100644 --- a/engine/schema/src/main/java/com/cloud/dc/dao/DataCenterDao.java +++ b/engine/schema/src/main/java/com/cloud/dc/dao/DataCenterDao.java @@ -115,4 +115,6 @@ public Integer getVlan() { List findByKeyword(String keyword); List listAllZones(); + + List listByIds(List ids); } diff --git a/engine/schema/src/main/java/com/cloud/dc/dao/DataCenterDaoImpl.java b/engine/schema/src/main/java/com/cloud/dc/dao/DataCenterDaoImpl.java index 0c75568cd812..491919bbca73 100644 --- a/engine/schema/src/main/java/com/cloud/dc/dao/DataCenterDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/dc/dao/DataCenterDaoImpl.java @@ -433,4 +433,14 @@ public List listAllZones() { return dcs; } + + @Override + public List listByIds(List ids) { + SearchBuilder idsSearch = createSearchBuilder(); + idsSearch.and("ids", idsSearch.entity().getId(), SearchCriteria.Op.IN); + idsSearch.done(); + SearchCriteria sc = idsSearch.create(); + sc.setParameters("ids", ids.toArray()); + return listBy(sc); + } } diff --git a/engine/schema/src/main/java/com/cloud/storage/SnapshotVO.java b/engine/schema/src/main/java/com/cloud/storage/SnapshotVO.java index ebfad6633ede..e9d6df85c2f2 100644 --- a/engine/schema/src/main/java/com/cloud/storage/SnapshotVO.java +++ b/engine/schema/src/main/java/com/cloud/storage/SnapshotVO.java @@ -16,9 +16,8 @@ // under the License. package com.cloud.storage; -import com.cloud.hypervisor.Hypervisor.HypervisorType; -import com.cloud.utils.db.GenericDao; -import com.google.gson.annotations.Expose; +import java.util.Date; +import java.util.UUID; import javax.persistence.Column; import javax.persistence.Entity; @@ -32,8 +31,9 @@ import org.apache.commons.lang3.builder.ToStringBuilder; import org.apache.commons.lang3.builder.ToStringStyle; -import java.util.Date; -import java.util.UUID; +import com.cloud.hypervisor.Hypervisor.HypervisorType; +import com.cloud.utils.db.GenericDao; +import com.google.gson.annotations.Expose; @Entity @Table(name = "snapshots") diff --git a/engine/schema/src/main/java/com/cloud/storage/SnapshotZoneVO.java b/engine/schema/src/main/java/com/cloud/storage/SnapshotZoneVO.java new file mode 100644 index 000000000000..82860defd6de --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/storage/SnapshotZoneVO.java @@ -0,0 +1,118 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.storage; + +import java.util.Date; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; +import javax.persistence.Temporal; +import javax.persistence.TemporalType; + +import org.apache.cloudstack.api.InternalIdentity; + +import com.cloud.utils.db.GenericDao; +import com.cloud.utils.db.GenericDaoBase; + +@Entity +@Table(name = "snapshot_zone_ref") +public class SnapshotZoneVO implements InternalIdentity { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + Long id; + + @Column(name = "zone_id") + private long zoneId; + + @Column(name = "snapshot_id") + private long snapshotId; + + @Column(name = GenericDaoBase.CREATED_COLUMN) + private Date created = null; + + @Column(name = "last_updated") + @Temporal(value = TemporalType.TIMESTAMP) + private Date lastUpdated = null; + + @Temporal(value = TemporalType.TIMESTAMP) + @Column(name = GenericDao.REMOVED_COLUMN) + private Date removed; + + protected SnapshotZoneVO() { + + } + + public SnapshotZoneVO(long zoneId, long snapshotId, Date lastUpdated) { + this.zoneId = zoneId; + this.snapshotId = snapshotId; + this.lastUpdated = lastUpdated; + } + + @Override + public long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public long getZoneId() { + return zoneId; + } + + public void setZoneId(long zoneId) { + this.zoneId = zoneId; + } + + public long getSnapshotId() { + return snapshotId; + } + + public void setSnapshotId(long snapshotId) { + this.snapshotId = snapshotId; + } + + public Date getCreated() { + return created; + } + + public void setCreated(Date created) { + this.created = created; + } + + public Date getLastUpdated() { + return lastUpdated; + } + + public void setLastUpdated(Date lastUpdated) { + this.lastUpdated = lastUpdated; + } + + public void setRemoved(Date removed) { + this.removed = removed; + } + + public Date getRemoved() { + return removed; + } + +} diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotZoneDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotZoneDao.java new file mode 100644 index 000000000000..186047c1aed1 --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotZoneDao.java @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.storage.dao; + +import java.util.List; + +import com.cloud.storage.SnapshotZoneVO; +import com.cloud.utils.db.GenericDao; + +public interface SnapshotZoneDao extends GenericDao { + SnapshotZoneVO findByZoneSnapshot(long zoneId, long templateId); + void addSnapshotToZone(long snapshotId, long zoneId); + void removeSnapshotFromZone(long snapshotId, long zoneId); + void removeSnapshotFromZones(long snapshotId); + List listBySnapshot(long snapshotId); +} diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotZoneDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotZoneDaoImpl.java new file mode 100644 index 000000000000..1ed8a547a105 --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotZoneDaoImpl.java @@ -0,0 +1,84 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.storage.dao; + +import java.util.Date; +import java.util.List; + +import org.apache.log4j.Logger; + +import com.cloud.storage.SnapshotZoneVO; +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; + +public class SnapshotZoneDaoImpl extends GenericDaoBase implements SnapshotZoneDao { + public static final Logger s_logger = Logger.getLogger(SnapshotZoneDaoImpl.class.getName()); + protected final SearchBuilder ZoneSnapshotSearch; + + public SnapshotZoneDaoImpl() { + + ZoneSnapshotSearch = createSearchBuilder(); + ZoneSnapshotSearch.and("zone_id", ZoneSnapshotSearch.entity().getZoneId(), SearchCriteria.Op.EQ); + ZoneSnapshotSearch.and("snapshot_id", ZoneSnapshotSearch.entity().getSnapshotId(), SearchCriteria.Op.EQ); + ZoneSnapshotSearch.done(); + } + + @Override + public SnapshotZoneVO findByZoneSnapshot(long zoneId, long snapshotId) { + SearchCriteria sc = ZoneSnapshotSearch.create(); + sc.setParameters("zone_id", zoneId); + sc.setParameters("snapshot_id", snapshotId); + return findOneBy(sc); + } + + @Override + public void addSnapshotToZone(long snapshotId, long zoneId) { + SnapshotZoneVO snapshotZone = findByZoneSnapshot(zoneId, snapshotId); + if (snapshotZone == null) { + snapshotZone = new SnapshotZoneVO(zoneId, snapshotId, new Date()); + persist(snapshotZone); + } else { + snapshotZone.setRemoved(GenericDaoBase.DATE_TO_NULL); + snapshotZone.setLastUpdated(new Date()); + update(snapshotZone.getId(), snapshotZone); + } + } + + @Override + public void removeSnapshotFromZone(long snapshotId, long zoneId) { + SearchCriteria sc = ZoneSnapshotSearch.create(); + sc.setParameters("zone_id", zoneId); + sc.setParameters("snapshot_id", snapshotId); + remove(sc); + } + + @Override + public void removeSnapshotFromZones(long snapshotId) { + SearchCriteria sc = ZoneSnapshotSearch.create(); + sc.setParameters("snapshot_id", snapshotId); + remove(sc); + } + + @Override + public List listBySnapshot(long snapshotId) { + SearchCriteria sc = ZoneSnapshotSearch.create(); + sc.setParameters("snapshot_id", snapshotId); + return listBy(sc); + } +} diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java index 2ce15894228d..1ddde246f79f 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java @@ -23,6 +23,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; import com.cloud.storage.DataStoreRole; +import com.cloud.storage.VMTemplateStorageResourceAssoc; import com.cloud.utils.db.GenericDao; import com.cloud.utils.fsm.StateDao; @@ -33,15 +34,21 @@ public interface SnapshotDataStoreDao extends GenericDao listByStoreIdAndState(long id, ObjectInDataStoreStateMachine.State state); + List listBySnapshotIdAndState(long id, ObjectInDataStoreStateMachine.State state); + List listActiveOnCache(long id); void deletePrimaryRecordsForStore(long id, DataStoreRole role); SnapshotDataStoreVO findByStoreSnapshot(DataStoreRole role, long storeId, long snapshotId); + void removeBySnapshotStore(long snapshotId, long storeId, DataStoreRole role); + SnapshotDataStoreVO findParent(DataStoreRole role, Long storeId, Long volumeId); - SnapshotDataStoreVO findBySnapshot(long snapshotId, DataStoreRole role); + List listBySnapshot(long snapshotId, DataStoreRole role); + + List listReadyBySnapshot(long snapshotId, DataStoreRole role); SnapshotDataStoreVO findBySourceSnapshot(long snapshotId, DataStoreRole role); @@ -66,9 +73,7 @@ public interface SnapshotDataStoreDao extends GenericDao findByVolume(long snapshotId, long volumeId, DataStoreRole role); /** * List all snapshots in 'snapshot_store_ref' by volume and data store role. Therefore, it is possible to list all snapshots that are in the primary storage or in the secondary storage. @@ -85,10 +90,16 @@ public interface SnapshotDataStoreDao extends GenericDao listReadyByVolumeId(long volumeId); + + List listBySnasphotStoreDownloadStatus(long snapshotId, long storeId, VMTemplateStorageResourceAssoc.Status... status); + + SnapshotDataStoreVO findOneBySnapshotAndDatastoreRole(long snapshotId, DataStoreRole role); + + void updateDisplayForSnapshotStoreRole(long snapshotId, long storeId, DataStoreRole role, boolean display); } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImpl.java index 066a36ddff45..657551ae8b77 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImpl.java @@ -37,6 +37,7 @@ import com.cloud.hypervisor.Hypervisor; import com.cloud.storage.DataStoreRole; import com.cloud.storage.SnapshotVO; +import com.cloud.storage.VMTemplateStorageResourceAssoc; import com.cloud.storage.dao.SnapshotDao; import com.cloud.utils.db.DB; import com.cloud.utils.db.Filter; @@ -61,8 +62,10 @@ public class SnapshotDataStoreDaoImpl extends GenericDaoBase searchFilteringStoreIdEqStoreRoleEqStateNeqRefCntNeq; protected SearchBuilder searchFilteringStoreIdEqStateEqStoreRoleEqIdEqUpdateCountEqSnapshotIdEqVolumeIdEq; private SearchBuilder stateSearch; + private SearchBuilder idStateNeqSearch; protected SearchBuilder snapshotVOSearch; private SearchBuilder snapshotCreatedSearch; + private SearchBuilder storeSnapshotDownloadStatusSearch; protected static final List HYPERVISORS_SUPPORTING_SNAPSHOTS_CHAINING = List.of(Hypervisor.HypervisorType.XenServer); @@ -114,6 +117,12 @@ public boolean configure(String name, Map params) throws Configu stateSearch.and(STATE, stateSearch.entity().getState(), SearchCriteria.Op.IN); stateSearch.done(); + + idStateNeqSearch = createSearchBuilder(); + idStateNeqSearch.and(SNAPSHOT_ID, idStateNeqSearch.entity().getSnapshotId(), SearchCriteria.Op.EQ); + idStateNeqSearch.and(STATE, idStateNeqSearch.entity().getState(), SearchCriteria.Op.NEQ); + idStateNeqSearch.done(); + snapshotVOSearch = snapshotDao.createSearchBuilder(); snapshotVOSearch.and(VOLUME_ID, snapshotVOSearch.entity().getVolumeId(), SearchCriteria.Op.EQ); snapshotVOSearch.done(); @@ -123,6 +132,12 @@ public boolean configure(String name, Map params) throws Configu snapshotCreatedSearch.and(CREATED, snapshotCreatedSearch.entity().getCreated(), SearchCriteria.Op.BETWEEN); snapshotCreatedSearch.done(); + storeSnapshotDownloadStatusSearch = createSearchBuilder(); + storeSnapshotDownloadStatusSearch.and(SNAPSHOT_ID, storeSnapshotDownloadStatusSearch.entity().getSnapshotId(), SearchCriteria.Op.EQ); + storeSnapshotDownloadStatusSearch.and(STORE_ID, storeSnapshotDownloadStatusSearch.entity().getDataStoreId(), SearchCriteria.Op.EQ); + storeSnapshotDownloadStatusSearch.and("downloadState", storeSnapshotDownloadStatusSearch.entity().getDownloadState(), SearchCriteria.Op.IN); + storeSnapshotDownloadStatusSearch.done(); + return true; } @@ -179,6 +194,14 @@ public List listByStoreIdAndState(long id, ObjectInDataStor return listBy(sc); } + @Override + public List listBySnapshotIdAndState(long id, ObjectInDataStoreStateMachine.State state) { + SearchCriteria sc = searchFilteringStoreIdEqStateEqStoreRoleEqIdEqUpdateCountEqSnapshotIdEqVolumeIdEq.create(); + sc.setParameters(SNAPSHOT_ID, id); + sc.setParameters(STATE, state); + return listBy(sc); + } + @Override public void deletePrimaryRecordsForStore(long id, DataStoreRole role) { SearchCriteria sc = searchFilteringStoreIdEqStoreRoleEqStateNeqRefCntNeq.create(); @@ -203,6 +226,15 @@ public SnapshotDataStoreVO findByStoreSnapshot(DataStoreRole role, long storeId, return findOneBy(sc); } + @Override + public void removeBySnapshotStore(long snapshotId, long storeId, DataStoreRole role) { + SearchCriteria sc = searchFilteringStoreIdEqStateEqStoreRoleEqIdEqUpdateCountEqSnapshotIdEqVolumeIdEq.create(); + sc.setParameters(STORE_ID, storeId); + sc.setParameters(SNAPSHOT_ID, snapshotId); + sc.setParameters(STORE_ROLE, role); + remove(sc); + } + @Override public SnapshotDataStoreVO findLatestSnapshotForVolume(Long volumeId, DataStoreRole role) { return findOldestOrLatestSnapshotForVolume(volumeId, role, false); @@ -257,10 +289,16 @@ public SnapshotDataStoreVO findParent(DataStoreRole role, Long storeId, Long vol } @Override - public SnapshotDataStoreVO findBySnapshot(long snapshotId, DataStoreRole role) { + public List listBySnapshot(long snapshotId, DataStoreRole role) { + SearchCriteria sc = createSearchCriteriaBySnapshotIdAndStoreRole(snapshotId, role); + return listBy(sc); + } + + @Override + public List listReadyBySnapshot(long snapshotId, DataStoreRole role) { SearchCriteria sc = createSearchCriteriaBySnapshotIdAndStoreRole(snapshotId, role); sc.setParameters(STATE, State.Ready); - return findOneBy(sc); + return listBy(sc); } @Override @@ -279,26 +317,19 @@ public List listAllByVolumeAndDataStore(long volumeId, Data } @Override - public SnapshotDataStoreVO findByVolume(long volumeId, DataStoreRole role) { - SearchCriteria sc = searchFilteringStoreIdEqStateEqStoreRoleEqIdEqUpdateCountEqSnapshotIdEqVolumeIdEq.create(); - sc.setParameters(VOLUME_ID, volumeId); - sc.setParameters(STORE_ROLE, role); - return findOneBy(sc); - } - - @Override - public SnapshotDataStoreVO findByVolume(long snapshotId, long volumeId, DataStoreRole role) { + public List findByVolume(long snapshotId, long volumeId, DataStoreRole role) { SearchCriteria sc = searchFilteringStoreIdEqStateEqStoreRoleEqIdEqUpdateCountEqSnapshotIdEqVolumeIdEq.create(); sc.setParameters(SNAPSHOT_ID, snapshotId); sc.setParameters(VOLUME_ID, volumeId); sc.setParameters(STORE_ROLE, role); - return findOneBy(sc); + return listBy(sc); } @Override public List findBySnapshotId(long snapshotId) { - SearchCriteria sc = searchFilteringStoreIdEqStateEqStoreRoleEqIdEqUpdateCountEqSnapshotIdEqVolumeIdEq.create(); + SearchCriteria sc = idStateNeqSearch.create(); sc.setParameters(SNAPSHOT_ID, snapshotId); + sc.setParameters(STATE, State.Destroyed); return listBy(sc); } @@ -451,8 +482,8 @@ protected boolean isSnapshotChainingRequired(long volumeId) { } @Override - public boolean expungeReferenceBySnapshotIdAndDataStoreRole(long snapshotId, DataStoreRole dataStoreRole) { - SnapshotDataStoreVO snapshotDataStoreVo = findOneBy(createSearchCriteriaBySnapshotIdAndStoreRole(snapshotId, dataStoreRole)); + public boolean expungeReferenceBySnapshotIdAndDataStoreRole(long snapshotId, long storeId, DataStoreRole dataStoreRole) { + SnapshotDataStoreVO snapshotDataStoreVo = findByStoreSnapshot(dataStoreRole, storeId, snapshotId); return snapshotDataStoreVo == null || expunge(snapshotDataStoreVo.getId()); } @@ -463,4 +494,30 @@ public List listReadyByVolumeId(long volumeId) { sc.setParameters(STATE, State.Ready); return listBy(sc); } + + @Override + public List listBySnasphotStoreDownloadStatus(long snapshotId, long storeId, VMTemplateStorageResourceAssoc.Status... status) { + SearchCriteria sc = storeSnapshotDownloadStatusSearch.create(); + sc.setParameters("snapshot_id", snapshotId); + sc.setParameters("store_id", storeId); + sc.setParameters("downloadState", (Object[])status); + return search(sc, null); + } + + @Override + public SnapshotDataStoreVO findOneBySnapshotAndDatastoreRole(long snapshotId, DataStoreRole role) { + SearchCriteria sc = createSearchCriteriaBySnapshotIdAndStoreRole(snapshotId, role); + sc.setParameters(STATE, State.Ready); + return findOneBy(sc); + } + + @Override + public void updateDisplayForSnapshotStoreRole(long snapshotId, long storeId, DataStoreRole role, boolean display) { + SnapshotDataStoreVO ref = findByStoreSnapshot(role, storeId, snapshotId); + if (ref == null) { + return; + } + ref.setDisplay(display); + update(ref.getId(), ref); + } } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreVO.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreVO.java index f36216911b02..6f6ed4e08f27 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreVO.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreVO.java @@ -36,6 +36,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine.State; import com.cloud.storage.DataStoreRole; +import com.cloud.storage.VMTemplateStorageResourceAssoc; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.fsm.StateObject; @@ -95,6 +96,22 @@ public class SnapshotDataStoreVO implements StateObject + diff --git a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml index 04ec733594e2..c00bbc155678 100644 --- a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml +++ b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml @@ -176,6 +176,7 @@ + diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql b/engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql index f7920667210a..b13531615481 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql @@ -183,3 +183,116 @@ ALTER TABLE `cloud`.`kubernetes_cluster` MODIFY COLUMN `kubernetes_version_id` b -- Set removed state for all removed accounts UPDATE `cloud`.`account` SET state='removed' WHERE `removed` IS NOT NULL; + +-- Add table for snapshot zone reference +CREATE TABLE `cloud`.`snapshot_zone_ref` ( + `id` bigint unsigned NOT NULL auto_increment, + `zone_id` bigint unsigned NOT NULL, + `snapshot_id` bigint unsigned NOT NULL, + `created` DATETIME NOT NULL, + `last_updated` DATETIME, + `removed` datetime COMMENT 'date removed if not null', + PRIMARY KEY (`id`), + CONSTRAINT `fk_snapshot_zone_ref__zone_id` FOREIGN KEY `fk_snapshot_zone_ref__zone_id` (`zone_id`) REFERENCES `data_center` (`id`) ON DELETE CASCADE, + INDEX `i_snapshot_zone_ref__zone_id`(`zone_id`), + CONSTRAINT `fk_snapshot_zone_ref__snapshot_id` FOREIGN KEY `fk_snapshot_zone_ref__snapshot_id` (`snapshot_id`) REFERENCES `snapshots` (`id`) ON DELETE CASCADE, + INDEX `i_snapshot_zone_ref__snapshot_id`(`snapshot_id`), + INDEX `i_snapshot_zone_ref__removed`(`removed`) +) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8; + +-- Alter snapshot_store_ref table to add download related fields +ALTER TABLE `cloud`.`snapshot_store_ref` + ADD COLUMN `download_state` varchar(255) DEFAULT NULL COMMENT 'the state of the snapshot download' AFTER `volume_id`, + ADD COLUMN `download_pct` int unsigned DEFAULT NULL COMMENT 'the percentage of the snapshot download completed' AFTER `download_state`, + ADD COLUMN `error_str` varchar(255) DEFAULT NULL COMMENT 'the error message when the snapshot download occurs' AFTER `download_pct`, + ADD COLUMN `local_path` varchar(255) DEFAULT NULL COMMENT 'the path of the snapshot download' AFTER `error_str`, + ADD COLUMN `display` tinyint(1) unsigned NOT NULL DEFAULT 1 COMMENT '1 implies store reference is available for listing' AFTER `error_str`; + +-- Create snapshot_view +DROP VIEW IF EXISTS `cloud`.`snapshot_view`; +CREATE VIEW `cloud`.`snapshot_view` AS + SELECT + `snapshots`.`id` AS `id`, + `snapshots`.`uuid` AS `uuid`, + `snapshots`.`name` AS `name`, + `snapshots`.`status` AS `status`, + `snapshots`.`disk_offering_id` AS `disk_offering_id`, + `snapshots`.`snapshot_type` AS `snapshot_type`, + `snapshots`.`type_description` AS `type_description`, + `snapshots`.`size` AS `size`, + `snapshots`.`created` AS `created`, + `snapshots`.`removed` AS `removed`, + `snapshots`.`location_type` AS `location_type`, + `snapshots`.`hypervisor_type` AS `hypervisor_type`, + `account`.`id` AS `account_id`, + `account`.`uuid` AS `account_uuid`, + `account`.`account_name` AS `account_name`, + `account`.`type` AS `account_type`, + `domain`.`id` AS `domain_id`, + `domain`.`uuid` AS `domain_uuid`, + `domain`.`name` AS `domain_name`, + `domain`.`path` AS `domain_path`, + `projects`.`id` AS `project_id`, + `projects`.`uuid` AS `project_uuid`, + `projects`.`name` AS `project_name`, + `volumes`.`id` AS `volume_id`, + `volumes`.`uuid` AS `volume_uuid`, + `volumes`.`name` AS `volume_name`, + `volumes`.`volume_type` AS `volume_type`, + `volumes`.`size` AS `volume_size`, + `data_center`.`id` AS `data_center_id`, + `data_center`.`uuid` AS `data_center_uuid`, + `data_center`.`name` AS `data_center_name`, + `snapshot_store_ref`.`store_id` AS `store_id`, + IFNULL(`image_store`.`uuid`, `storage_pool`.`uuid`) AS `store_uuid`, + IFNULL(`image_store`.`name`, `storage_pool`.`name`) AS `store_name`, + `snapshot_store_ref`.`store_role` AS `store_role`, + `snapshot_store_ref`.`state` AS `store_state`, + `snapshot_store_ref`.`download_state` AS `download_state`, + `snapshot_store_ref`.`download_pct` AS `download_pct`, + `snapshot_store_ref`.`error_str` AS `error_str`, + `snapshot_store_ref`.`size` AS `store_size`, + `snapshot_store_ref`.`created` AS `created_on_store`, + `resource_tags`.`id` AS `tag_id`, + `resource_tags`.`uuid` AS `tag_uuid`, + `resource_tags`.`key` AS `tag_key`, + `resource_tags`.`value` AS `tag_value`, + `resource_tags`.`domain_id` AS `tag_domain_id`, + `domain`.`uuid` AS `tag_domain_uuid`, + `domain`.`name` AS `tag_domain_name`, + `resource_tags`.`account_id` AS `tag_account_id`, + `account`.`account_name` AS `tag_account_name`, + `resource_tags`.`resource_id` AS `tag_resource_id`, + `resource_tags`.`resource_uuid` AS `tag_resource_uuid`, + `resource_tags`.`resource_type` AS `tag_resource_type`, + `resource_tags`.`customer` AS `tag_customer`, + CONCAT(`snapshots`.`id`, + '_', + IFNULL(`snapshot_store_ref`.`store_role`, 'UNKNOWN'), + '_', + IFNULL(`snapshot_store_ref`.`store_id`, 0)) AS `snapshot_store_pair` + FROM + ((((((((((`snapshots` + JOIN `account` ON ((`account`.`id` = `snapshots`.`account_id`))) + JOIN `domain` ON ((`domain`.`id` = `account`.`domain_id`))) + LEFT JOIN `projects` ON ((`projects`.`project_account_id` = `account`.`id`))) + LEFT JOIN `volumes` ON ((`volumes`.`id` = `snapshots`.`volume_id`))) + LEFT JOIN `snapshot_store_ref` ON (((`snapshot_store_ref`.`snapshot_id` = `snapshots`.`id`) + AND (`snapshot_store_ref`.`state` != 'Destroyed') + AND (`snapshot_store_ref`.`display` = 1)))) + LEFT JOIN `image_store` ON ((ISNULL(`image_store`.`removed`) + AND (`snapshot_store_ref`.`store_role` = 'Image') + AND (`snapshot_store_ref`.`store_id` IS NOT NULL) + AND (`image_store`.`id` = `snapshot_store_ref`.`store_id`)))) + LEFT JOIN `storage_pool` ON ((ISNULL(`storage_pool`.`removed`) + AND (`snapshot_store_ref`.`store_role` = 'Primary') + AND (`snapshot_store_ref`.`store_id` IS NOT NULL) + AND (`storage_pool`.`id` = `snapshot_store_ref`.`store_id`)))) + LEFT JOIN `snapshot_zone_ref` ON (((`snapshot_zone_ref`.`snapshot_id` = `snapshots`.`id`) + AND ISNULL(`snapshot_store_ref`.`store_id`) + AND ISNULL(`snapshot_zone_ref`.`removed`)))) + LEFT JOIN `data_center` ON (((`image_store`.`data_center_id` = `data_center`.`id`) + OR (`storage_pool`.`data_center_id` = `data_center`.`id`) + OR (`snapshot_zone_ref`.`zone_id` = `data_center`.`id`)))) + LEFT JOIN `resource_tags` ON ((`resource_tags`.`resource_id` = `snapshots`.`id`) + AND (`resource_tags`.`resource_type` = 'Snapshot'))); diff --git a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/SecondaryStorageServiceImpl.java b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/SecondaryStorageServiceImpl.java index 1a6a31fafcb7..3557921a8936 100644 --- a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/SecondaryStorageServiceImpl.java +++ b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/SecondaryStorageServiceImpl.java @@ -218,7 +218,7 @@ protected Void migrateDataCallBack(AsyncCallbackDispatcher[] getConfigKeys() { return new ConfigKey[] { ImageStoreAllocationAlgorithm }; } + + @Override + public long getImageStoreZoneId(long dataStoreId) { + ImageStoreVO dataStore = dataStoreDao.findById(dataStoreId); + return dataStore.getDataCenterId(); + } } diff --git a/engine/storage/image/src/test/java/org/apache/cloudstack/storage/image/manager/ImageStoreProviderManagerImplTest.java b/engine/storage/image/src/test/java/org/apache/cloudstack/storage/image/manager/ImageStoreProviderManagerImplTest.java new file mode 100644 index 000000000000..c04620347906 --- /dev/null +++ b/engine/storage/image/src/test/java/org/apache/cloudstack/storage/image/manager/ImageStoreProviderManagerImplTest.java @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.image.manager; + +import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; +import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +public class ImageStoreProviderManagerImplTest { + + @Mock + ImageStoreDao imageStoreDao; + + @InjectMocks + ImageStoreProviderManagerImpl imageStoreProviderManager = new ImageStoreProviderManagerImpl(); + @Test + public void testGetImageStoreZoneId() { + final long storeId = 1L; + final long zoneId = 1L; + ImageStoreVO imageStoreVO = Mockito.mock(ImageStoreVO.class); + Mockito.when(imageStoreVO.getDataCenterId()).thenReturn(zoneId); + Mockito.when(imageStoreDao.findById(storeId)).thenReturn(imageStoreVO); + long value = imageStoreProviderManager.getImageStoreZoneId(storeId); + Assert.assertEquals(zoneId, value); + } +} diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/CephSnapshotStrategy.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/CephSnapshotStrategy.java index 02672f29f503..19b3fc87f4e3 100644 --- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/CephSnapshotStrategy.java +++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/CephSnapshotStrategy.java @@ -21,7 +21,6 @@ import javax.inject.Inject; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; -import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotStrategy.SnapshotOperation; import org.apache.cloudstack.engine.subsystem.api.storage.StrategyPriority; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; @@ -48,7 +47,7 @@ public class CephSnapshotStrategy extends StorageSystemSnapshotStrategy { private static final Logger s_logger = Logger.getLogger(CephSnapshotStrategy.class); @Override - public StrategyPriority canHandle(Snapshot snapshot, SnapshotOperation op) { + public StrategyPriority canHandle(Snapshot snapshot, Long zoneId, SnapshotOperation op) { long volumeId = snapshot.getVolumeId(); VolumeVO volumeVO = volumeDao.findByIdIncludingRemoved(volumeId); boolean baseVolumeExists = volumeVO.getRemoved() == null; @@ -56,7 +55,7 @@ public StrategyPriority canHandle(Snapshot snapshot, SnapshotOperation op) { return StrategyPriority.CANT_HANDLE; } - if (!isSnapshotStoredOnRbdStoragePool(snapshot)) { + if (!isSnapshotStoredOnRbdStoragePoolAndOperationForSameZone(snapshot, zoneId)) { return StrategyPriority.CANT_HANDLE; } @@ -81,12 +80,18 @@ public boolean revertSnapshot(SnapshotInfo snapshotInfo) { return true; } - protected boolean isSnapshotStoredOnRbdStoragePool(Snapshot snapshot) { - SnapshotDataStoreVO snapshotStore = snapshotStoreDao.findBySnapshot(snapshot.getId(), DataStoreRole.Primary); + protected boolean isSnapshotStoredOnRbdStoragePoolAndOperationForSameZone(Snapshot snapshot, Long zoneId) { + SnapshotDataStoreVO snapshotStore = snapshotStoreDao.findOneBySnapshotAndDatastoreRole(snapshot.getId(), DataStoreRole.Primary); if (snapshotStore == null) { return false; } StoragePoolVO storagePoolVO = primaryDataStoreDao.findById(snapshotStore.getDataStoreId()); - return storagePoolVO != null && storagePoolVO.getPoolType() == StoragePoolType.RBD; + if (storagePoolVO == null) { + return false; + } + if (zoneId != null && !zoneId.equals(storagePoolVO.getDataCenterId())) { + return false; + } + return storagePoolVO.getPoolType() == StoragePoolType.RBD; } } diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/DefaultSnapshotStrategy.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/DefaultSnapshotStrategy.java index 85e0a02c5f7b..59f5b7c86827 100644 --- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/DefaultSnapshotStrategy.java +++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/DefaultSnapshotStrategy.java @@ -16,18 +16,13 @@ // under the License. package org.apache.cloudstack.storage.snapshot; +import java.util.ArrayList; import java.util.Arrays; -import java.util.LinkedHashMap; import java.util.List; -import java.util.Map; +import java.util.Objects; import javax.inject.Inject; -import com.cloud.storage.VolumeDetailVO; -import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.BooleanUtils; -import org.apache.log4j.Logger; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine.Event; @@ -41,11 +36,15 @@ import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.framework.jobs.AsyncJob; import org.apache.cloudstack.storage.command.CreateObjectAnswer; +import org.apache.cloudstack.storage.datastore.PrimaryDataStoreImpl; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; -import org.apache.cloudstack.storage.datastore.PrimaryDataStoreImpl; import org.apache.cloudstack.storage.to.SnapshotObjectTO; import org.apache.cloudstack.utils.identity.ManagementServerNode; +import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.BooleanUtils; +import org.apache.log4j.Logger; import com.cloud.agent.api.to.DataTO; import com.cloud.event.EventTypes; @@ -57,28 +56,28 @@ import com.cloud.storage.DataStoreRole; import com.cloud.storage.Snapshot; import com.cloud.storage.SnapshotVO; +import com.cloud.storage.Storage.ImageFormat; +import com.cloud.storage.Storage.StoragePoolType; import com.cloud.storage.StoragePool; import com.cloud.storage.StoragePoolStatus; import com.cloud.storage.Volume; +import com.cloud.storage.VolumeDetailVO; import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.SnapshotDao; import com.cloud.storage.dao.SnapshotDetailsDao; +import com.cloud.storage.dao.SnapshotZoneDao; import com.cloud.storage.dao.VolumeDao; import com.cloud.storage.dao.VolumeDetailsDao; -import com.cloud.utils.db.Transaction; -import com.cloud.utils.db.TransactionCallbackNoReturn; -import com.cloud.utils.db.TransactionStatus; import com.cloud.storage.snapshot.SnapshotManager; -import com.cloud.storage.Storage.ImageFormat; -import com.cloud.storage.Storage.StoragePoolType; import com.cloud.utils.NumbersUtil; import com.cloud.utils.db.DB; +import com.cloud.utils.db.Transaction; +import com.cloud.utils.db.TransactionCallbackNoReturn; +import com.cloud.utils.db.TransactionStatus; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.fsm.NoTransitionException; public class DefaultSnapshotStrategy extends SnapshotStrategyBase { - private static final String SECONDARY_STORAGE_SNAPSHOT_ENTRY_IDENTIFIER = "secondary storage"; - private static final String PRIMARY_STORAGE_SNAPSHOT_ENTRY_IDENTIFIER = "primary storage"; private static final Logger s_logger = Logger.getLogger(DefaultSnapshotStrategy.class); @@ -100,6 +99,18 @@ public class DefaultSnapshotStrategy extends SnapshotStrategyBase { private SnapshotDetailsDao _snapshotDetailsDao; @Inject VolumeDetailsDao _volumeDetailsDaoImpl; + @Inject + SnapshotZoneDao snapshotZoneDao; + + public SnapshotDataStoreVO getSnapshotImageStoreRef(long snapshotId, long zoneId) { + List snaps = snapshotStoreDao.listReadyBySnapshot(snapshotId, DataStoreRole.Image); + for (SnapshotDataStoreVO ref : snaps) { + if (zoneId == dataStoreMgr.getStoreZoneId(ref.getDataStoreId(), ref.getRole())) { + return ref; + } + } + return null; + } @Override public SnapshotInfo backupSnapshot(SnapshotInfo snapshot) { @@ -107,7 +118,8 @@ public SnapshotInfo backupSnapshot(SnapshotInfo snapshot) { if (parentSnapshot != null && snapshot.getPath().equalsIgnoreCase(parentSnapshot.getPath())) { // don't need to backup this snapshot - SnapshotDataStoreVO parentSnapshotOnBackupStore = snapshotStoreDao.findBySnapshot(parentSnapshot.getId(), DataStoreRole.Image); + SnapshotDataStoreVO parentSnapshotOnBackupStore = getSnapshotImageStoreRef(parentSnapshot.getId(), + dataStoreMgr.getStoreZoneId(parentSnapshot.getDataStore().getId(), parentSnapshot.getDataStore().getRole())); if (parentSnapshotOnBackupStore != null && parentSnapshotOnBackupStore.getState() == State.Ready) { DataStore store = dataStoreMgr.getDataStore(parentSnapshotOnBackupStore.getDataStoreId(), parentSnapshotOnBackupStore.getRole()); @@ -159,7 +171,7 @@ public SnapshotInfo backupSnapshot(SnapshotInfo snapshot) { if (prevBackupId == 0) { break; } - parentSnapshotOnBackupStore = snapshotStoreDao.findBySnapshot(prevBackupId, DataStoreRole.Image); + parentSnapshotOnBackupStore = getSnapshotImageStoreRef(prevBackupId, volume.getDataCenterId()); if (parentSnapshotOnBackupStore == null) { break; } @@ -181,20 +193,19 @@ public SnapshotInfo backupSnapshot(SnapshotInfo snapshot) { return snapshotSvr.backupSnapshot(snapshot); } - private final List snapshotStatesAbleToDeleteSnapshot = Arrays.asList(Snapshot.State.Destroying, Snapshot.State.Destroyed, Snapshot.State.Error); - - protected boolean deleteSnapshotChain(SnapshotInfo snapshot, String storage) { + protected boolean deleteSnapshotChain(SnapshotInfo snapshot, String storageToString) { DataTO snapshotTo = snapshot.getTO(); s_logger.debug(String.format("Deleting %s chain of snapshots.", snapshotTo)); boolean result = false; boolean resultIsSet = false; + final List snapshotStatesAbleToDeleteSnapshot = Arrays.asList(Snapshot.State.BackedUp, Snapshot.State.Destroying, Snapshot.State.Destroyed, Snapshot.State.Error); try { while (snapshot != null && snapshotStatesAbleToDeleteSnapshot.contains(snapshot.getState())) { SnapshotInfo child = snapshot.getChild(); if (child != null) { - s_logger.debug(String.format("Snapshot [%s] has child [%s], not deleting it on the storage [%s]", snapshotTo, child.getTO(), storage)); + s_logger.debug(String.format("Snapshot [%s] has child [%s], not deleting it on the storage [%s]", snapshotTo, child.getTO(), storageToString)); break; } @@ -207,8 +218,6 @@ protected boolean deleteSnapshotChain(SnapshotInfo snapshot, String storage) { //NOTE: if both snapshots share the same path, it's for xenserver's empty delta snapshot. We can't delete the snapshot on the backend, as parent snapshot still reference to it //Instead, mark it as destroyed in the db. s_logger.debug(String.format("Snapshot [%s] is an empty delta snapshot; therefore, we will only mark it as destroyed in the database.", snapshotTo)); - snapshot.processEvent(Event.DestroyRequested); - snapshot.processEvent(Event.OperationSuccessed); deleted = true; if (!resultIsSet) { result = true; @@ -233,22 +242,25 @@ protected boolean deleteSnapshotChain(SnapshotInfo snapshot, String storage) { resultIsSet = true; } } catch (Exception e) { - s_logger.error(String.format("Failed to delete snapshot [%s] on storage [%s] due to [%s].", snapshotTo, storage, e.getMessage()), e); + s_logger.error(String.format("Failed to delete snapshot [%s] on storage [%s] due to [%s].", snapshotTo, storageToString, e.getMessage()), e); } } snapshot = parent; } } catch (Exception e) { - s_logger.error(String.format("Failed to delete snapshot [%s] on storage [%s] due to [%s].", snapshotTo, storage, e.getMessage()), e); + s_logger.error(String.format("Failed to delete snapshot [%s] on storage [%s] due to [%s].", snapshotTo, storageToString, e.getMessage()), e); } return result; } @Override - public boolean deleteSnapshot(Long snapshotId) { + public boolean deleteSnapshot(Long snapshotId, Long zoneId) { SnapshotVO snapshotVO = snapshotDao.findById(snapshotId); + if (zoneId != null && List.of(Snapshot.State.Allocated, Snapshot.State.CreatedOnPrimary).contains(snapshotVO.getState())) { + throw new InvalidParameterValueException(String.format("Snapshot in %s can not be deleted for a zone", snapshotVO.getState())); + } if (snapshotVO.getState() == Snapshot.State.Allocated) { snapshotDao.remove(snapshotId); return true; @@ -260,10 +272,21 @@ public boolean deleteSnapshot(Long snapshotId) { if (Snapshot.State.Error.equals(snapshotVO.getState())) { List storeRefs = snapshotStoreDao.findBySnapshotId(snapshotId); + List deletedRefs = new ArrayList<>(); for (SnapshotDataStoreVO ref : storeRefs) { - snapshotStoreDao.expunge(ref.getId()); + boolean refZoneIdMatch = false; + if (zoneId != null) { + Long refZoneId = dataStoreMgr.getStoreZoneId(ref.getDataStoreId(), ref.getRole()); + refZoneIdMatch = zoneId.equals(refZoneId); + } + if (zoneId == null || refZoneIdMatch) { + snapshotStoreDao.expunge(ref.getId()); + deletedRefs.add(ref.getId()); + } + } + if (deletedRefs.size() == storeRefs.size()) { + snapshotDao.remove(snapshotId); } - snapshotDao.remove(snapshotId); return true; } @@ -278,20 +301,26 @@ public boolean deleteSnapshot(Long snapshotId) { throw new InvalidParameterValueException("Can't delete snapshotshot " + snapshotId + " due to it is in " + snapshotVO.getState() + " Status"); } - return destroySnapshotEntriesAndFiles(snapshotVO); + return destroySnapshotEntriesAndFiles(snapshotVO, zoneId); } /** * Destroys the snapshot entries and files on both primary and secondary storage (if it exists). * @return true if destroy successfully, else false. */ - protected boolean destroySnapshotEntriesAndFiles(SnapshotVO snapshotVo) { - if (!deleteSnapshotInfos(snapshotVo)) { + protected boolean destroySnapshotEntriesAndFiles(SnapshotVO snapshotVo, Long zoneId) { + if (!deleteSnapshotInfos(snapshotVo, zoneId)) { return false; } - + if (zoneId != null) { + snapshotZoneDao.removeSnapshotFromZone(snapshotVo.getId(), zoneId); + } else { + snapshotZoneDao.removeSnapshotFromZones(snapshotVo.getId()); + } + if (CollectionUtils.isNotEmpty(retrieveSnapshotEntries(snapshotVo.getId(), null))) { + return true; + } updateSnapshotToDestroyed(snapshotVo); - return true; } @@ -303,12 +332,12 @@ protected void updateSnapshotToDestroyed(SnapshotVO snapshotVo) { snapshotDao.update(snapshotVo.getId(), snapshotVo); } - protected boolean deleteSnapshotInfos(SnapshotVO snapshotVo) { - Map snapshotInfos = retrieveSnapshotEntries(snapshotVo.getId()); + protected boolean deleteSnapshotInfos(SnapshotVO snapshotVo, Long zoneId) { + List snapshotInfos = retrieveSnapshotEntries(snapshotVo.getId(), zoneId); boolean result = false; - for (var infoEntry : snapshotInfos.entrySet()) { - if (BooleanUtils.toBooleanDefaultIfNull(deleteSnapshotInfo(infoEntry.getValue(), infoEntry.getKey(), snapshotVo), false)) { + for (var snapshotInfo : snapshotInfos) { + if (BooleanUtils.toBooleanDefaultIfNull(deleteSnapshotInfo(snapshotInfo, snapshotVo), false)) { result = true; } } @@ -320,50 +349,53 @@ protected boolean deleteSnapshotInfos(SnapshotVO snapshotVo) { * Destroys the snapshot entry and file. * @return true if destroy successfully, else false. */ - protected Boolean deleteSnapshotInfo(SnapshotInfo snapshotInfo, String storage, SnapshotVO snapshotVo) { - if (snapshotInfo == null) { - s_logger.debug(String.format("Could not find %s entry on %s. Skipping deletion on %s.", snapshotVo, storage, storage)); - return SECONDARY_STORAGE_SNAPSHOT_ENTRY_IDENTIFIER.equals(storage) ? null : true; - } - + protected Boolean deleteSnapshotInfo(SnapshotInfo snapshotInfo, SnapshotVO snapshotVo) { DataStore dataStore = snapshotInfo.getDataStore(); - String storageToString = String.format("%s {uuid: \"%s\", name: \"%s\"}", storage, dataStore.getUuid(), dataStore.getName()); - + String storageToString = String.format("%s {uuid: \"%s\", name: \"%s\"}", dataStore.getRole().name(), dataStore.getUuid(), dataStore.getName()); + List snapshotStoreRefs = snapshotStoreDao.findBySnapshotId(snapshotVo.getId()); + boolean isLastSnapshotRef = CollectionUtils.isEmpty(snapshotStoreRefs) || snapshotStoreRefs.size() == 1; try { SnapshotObject snapshotObject = castSnapshotInfoToSnapshotObject(snapshotInfo); - snapshotObject.processEvent(Snapshot.Event.DestroyRequested); - - if (SECONDARY_STORAGE_SNAPSHOT_ENTRY_IDENTIFIER.equals(storage)) { - + if (isLastSnapshotRef) { + snapshotObject.processEvent(Snapshot.Event.DestroyRequested); + } + if (!DataStoreRole.Primary.equals(dataStore.getRole())) { verifyIfTheSnapshotIsBeingUsedByAnyVolume(snapshotObject); - if (deleteSnapshotChain(snapshotInfo, storageToString)) { s_logger.debug(String.format("%s was deleted on %s. We will mark the snapshot as destroyed.", snapshotVo, storageToString)); } else { s_logger.debug(String.format("%s was not deleted on %s; however, we will mark the snapshot as destroyed for future garbage collecting.", snapshotVo, storageToString)); } - - snapshotObject.processEvent(Snapshot.Event.OperationSucceeded); + snapshotStoreDao.updateDisplayForSnapshotStoreRole(snapshotVo.getId(), dataStore.getId(), dataStore.getRole(), false); + if (isLastSnapshotRef) { + snapshotObject.processEvent(Snapshot.Event.OperationSucceeded); + } return true; - } else if (deleteSnapshotInPrimaryStorage(snapshotInfo, snapshotVo, storageToString, snapshotObject)) { + } else if (deleteSnapshotInPrimaryStorage(snapshotInfo, snapshotVo, storageToString, snapshotObject, isLastSnapshotRef)) { + snapshotStoreDao.updateDisplayForSnapshotStoreRole(snapshotVo.getId(), dataStore.getId(), dataStore.getRole(), false); return true; } - s_logger.debug(String.format("Failed to delete %s on %s.", snapshotVo, storageToString)); - snapshotObject.processEvent(Snapshot.Event.OperationFailed); + if (isLastSnapshotRef) { + snapshotObject.processEvent(Snapshot.Event.OperationFailed); + } } catch (NoTransitionException ex) { s_logger.warn(String.format("Failed to delete %s on %s due to %s.", snapshotVo, storageToString, ex.getMessage()), ex); } - return false; } - protected boolean deleteSnapshotInPrimaryStorage(SnapshotInfo snapshotInfo, SnapshotVO snapshotVo, String storageToString, SnapshotObject snapshotObject) throws NoTransitionException { + protected boolean deleteSnapshotInPrimaryStorage(SnapshotInfo snapshotInfo, SnapshotVO snapshotVo, + String storageToString, SnapshotObject snapshotObject, boolean isLastSnapshotRef) throws NoTransitionException { try { if (snapshotSvr.deleteSnapshot(snapshotInfo)) { - snapshotObject.processEvent(Snapshot.Event.OperationSucceeded); - s_logger.debug(String.format("%s was deleted on %s. We will mark the snapshot as destroyed.", snapshotVo, storageToString)); + String msg = String.format("%s was deleted on %s.", snapshotVo, storageToString); + if (isLastSnapshotRef) { + msg = String.format("%s We will mark the snapshot as destroyed.", msg); + snapshotObject.processEvent(Snapshot.Event.OperationSucceeded); + } + s_logger.debug(msg); return true; } } catch (CloudRuntimeException ex) { @@ -396,18 +428,15 @@ protected SnapshotObject castSnapshotInfoToSnapshotObject(SnapshotInfo snapshotI /** * Retrieves the snapshot infos on primary and secondary storage. * @param snapshotId The snapshot to retrieve the infos. - * @return A map of snapshot infos. + * @return A list of snapshot infos. */ - protected Map retrieveSnapshotEntries(long snapshotId) { - Map snapshotInfos = new LinkedHashMap<>(); - snapshotInfos.put(SECONDARY_STORAGE_SNAPSHOT_ENTRY_IDENTIFIER, snapshotDataFactory.getSnapshot(snapshotId, DataStoreRole.Image, false)); - snapshotInfos.put(PRIMARY_STORAGE_SNAPSHOT_ENTRY_IDENTIFIER, snapshotDataFactory.getSnapshot(snapshotId, DataStoreRole.Primary, false)); - return snapshotInfos; + protected List retrieveSnapshotEntries(long snapshotId, Long zoneId) { + return snapshotDataFactory.getSnapshots(snapshotId, zoneId); } @Override public boolean revertSnapshot(SnapshotInfo snapshot) { - if (canHandle(snapshot, SnapshotOperation.REVERT) == StrategyPriority.CANT_HANDLE) { + if (canHandle(snapshot, null, SnapshotOperation.REVERT) == StrategyPriority.CANT_HANDLE) { throw new CloudRuntimeException("Reverting not supported. Create a template or volume based on the snapshot instead."); } @@ -542,19 +571,31 @@ public void doInTransactionWithoutResult(TransactionStatus status) { } @Override - public StrategyPriority canHandle(Snapshot snapshot, SnapshotOperation op) { + public StrategyPriority canHandle(Snapshot snapshot, Long zoneId, SnapshotOperation op) { if (SnapshotOperation.REVERT.equals(op)) { long volumeId = snapshot.getVolumeId(); VolumeVO volumeVO = volumeDao.findById(volumeId); - if (volumeVO != null && ImageFormat.QCOW2.equals(volumeVO.getFormat())) { + if (isSnapshotStoredOnSameZoneStoreForQCOW2Volume(snapshot, volumeVO)) { return StrategyPriority.DEFAULT; } return StrategyPriority.CANT_HANDLE; } - + if (zoneId != null && SnapshotOperation.DELETE.equals(op)) { + s_logger.debug(String.format("canHandle for zone ID: %d, operation: %s - %s", zoneId, op, StrategyPriority.DEFAULT)); + } return StrategyPriority.DEFAULT; } + protected boolean isSnapshotStoredOnSameZoneStoreForQCOW2Volume(Snapshot snapshot, VolumeVO volumeVO) { + if (volumeVO == null || !ImageFormat.QCOW2.equals(volumeVO.getFormat())) { + return false; + } + List snapshotStores = snapshotStoreDao.listBySnapshotIdAndState(snapshot.getId(), State.Ready); + return CollectionUtils.isNotEmpty(snapshotStores) && + snapshotStores.stream().anyMatch(s -> Objects.equals( + dataStoreMgr.getStoreZoneId(s.getDataStoreId(), s.getRole()), volumeVO.getDataCenterId())); + } + } diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/ScaleIOSnapshotStrategy.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/ScaleIOSnapshotStrategy.java index dfe475004f78..3dee4f4aa94f 100644 --- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/ScaleIOSnapshotStrategy.java +++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/ScaleIOSnapshotStrategy.java @@ -45,7 +45,7 @@ public class ScaleIOSnapshotStrategy extends StorageSystemSnapshotStrategy { private static final Logger LOG = Logger.getLogger(ScaleIOSnapshotStrategy.class); @Override - public StrategyPriority canHandle(Snapshot snapshot, SnapshotOperation op) { + public StrategyPriority canHandle(Snapshot snapshot, Long zoneId, SnapshotOperation op) { long volumeId = snapshot.getVolumeId(); VolumeVO volumeVO = volumeDao.findByIdIncludingRemoved(volumeId); boolean baseVolumeExists = volumeVO.getRemoved() == null; @@ -53,7 +53,7 @@ public StrategyPriority canHandle(Snapshot snapshot, SnapshotOperation op) { return StrategyPriority.CANT_HANDLE; } - if (!isSnapshotStoredOnScaleIOStoragePool(snapshot)) { + if (!isSnapshotStoredOnScaleIOStoragePoolAndOperationForSameZone(snapshot, zoneId)) { return StrategyPriority.CANT_HANDLE; } @@ -82,12 +82,18 @@ public boolean revertSnapshot(SnapshotInfo snapshotInfo) { return true; } - protected boolean isSnapshotStoredOnScaleIOStoragePool(Snapshot snapshot) { - SnapshotDataStoreVO snapshotStore = snapshotStoreDao.findBySnapshot(snapshot.getId(), DataStoreRole.Primary); + protected boolean isSnapshotStoredOnScaleIOStoragePoolAndOperationForSameZone(Snapshot snapshot, Long zoneId) { + SnapshotDataStoreVO snapshotStore = snapshotStoreDao.findOneBySnapshotAndDatastoreRole(snapshot.getId(), DataStoreRole.Primary); if (snapshotStore == null) { return false; } StoragePoolVO storagePoolVO = primaryDataStoreDao.findById(snapshotStore.getDataStoreId()); - return storagePoolVO != null && storagePoolVO.getPoolType() == Storage.StoragePoolType.PowerFlex; + if (storagePoolVO == null) { + return false; + } + if (zoneId != null && !zoneId.equals(storagePoolVO.getDataCenterId())) { + return false; + } + return storagePoolVO.getPoolType() == Storage.StoragePoolType.PowerFlex; } } diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotDataFactoryImpl.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotDataFactoryImpl.java index d894d7953ffa..fc5e61ef710f 100644 --- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotDataFactoryImpl.java +++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotDataFactoryImpl.java @@ -64,7 +64,7 @@ public SnapshotInfo getSnapshot(DataObject obj, DataStore store) { } @Override - public List getSnapshots(long volumeId, DataStoreRole role) { + public List getSnapshotsForVolumeAndStoreRole(long volumeId, DataStoreRole role) { List allSnapshotsFromVolumeAndDataStore = snapshotStoreDao.listAllByVolumeAndDataStore(volumeId, role); if (CollectionUtils.isEmpty(allSnapshotsFromVolumeAndDataStore)) { return new ArrayList<>(); @@ -84,23 +84,90 @@ public List getSnapshots(long volumeId, DataStoreRole role) { } @Override - public SnapshotInfo getSnapshot(long snapshotId, DataStoreRole role) { - return getSnapshot(snapshotId, role, true); + public List getSnapshots(long snapshotId, Long zoneId) { + SnapshotVO snapshot = snapshotDao.findById(snapshotId); + if (snapshot == null) { //snapshot may have been removed; + return new ArrayList<>(); + } + List allSnapshotsAndDataStore = snapshotStoreDao.findBySnapshotId(snapshotId); + if (CollectionUtils.isEmpty(allSnapshotsAndDataStore)) { + return new ArrayList<>(); + } + List infos = new ArrayList<>(); + for (SnapshotDataStoreVO snapshotDataStoreVO : allSnapshotsAndDataStore) { + Long entryZoneId = storeMgr.getStoreZoneId(snapshotDataStoreVO.getDataStoreId(), snapshotDataStoreVO.getRole()); + if (zoneId != null && !zoneId.equals(entryZoneId)) { + continue; + } + DataStore store = storeMgr.getDataStore(snapshotDataStoreVO.getDataStoreId(), snapshotDataStoreVO.getRole()); + SnapshotObject info = SnapshotObject.getSnapshotObject(snapshot, store); + + infos.add(info); + } + return infos; } + + @Override - public SnapshotInfo getSnapshot(long snapshotId, DataStoreRole role, boolean retrieveAnySnapshotFromVolume) { + public SnapshotInfo getSnapshot(long snapshotId, long storeId, DataStoreRole role) { SnapshotVO snapshot = snapshotDao.findById(snapshotId); if (snapshot == null) { return null; } - SnapshotDataStoreVO snapshotStore = snapshotStoreDao.findBySnapshot(snapshotId, role); + SnapshotDataStoreVO snapshotStore = snapshotStoreDao.findByStoreSnapshot(role, storeId, snapshotId); + if (snapshotStore == null) { + return null; + } + DataStore store = storeMgr.getDataStore(snapshotStore.getDataStoreId(), role); + return SnapshotObject.getSnapshotObject(snapshot, store); + } + + @Override + public SnapshotInfo getSnapshotWithRoleAndZone(long snapshotId, DataStoreRole role, long zoneId) { + return getSnapshot(snapshotId, role, zoneId, true); + } + + @Override + public SnapshotInfo getSnapshotOnPrimaryStore(long snapshotId) { + SnapshotVO snapshot = snapshotDao.findById(snapshotId); + if (snapshot == null) { + return null; + } + SnapshotDataStoreVO snapshotStore = snapshotStoreDao.findOneBySnapshotAndDatastoreRole(snapshotId, DataStoreRole.Primary); + if (snapshotStore == null) { + return null; + } + DataStore store = storeMgr.getDataStore(snapshotStore.getDataStoreId(), snapshotStore.getRole()); + SnapshotObject so = SnapshotObject.getSnapshotObject(snapshot, store); + return so; + } + + @Override + public SnapshotInfo getSnapshot(long snapshotId, DataStoreRole role, long zoneId, boolean retrieveAnySnapshotFromVolume) { + SnapshotVO snapshot = snapshotDao.findById(snapshotId); + if (snapshot == null) { + return null; + } + List snapshotStores = snapshotStoreDao.listReadyBySnapshot(snapshotId, role); + SnapshotDataStoreVO snapshotStore = null; + for (SnapshotDataStoreVO ref : snapshotStores) { + if (zoneId == storeMgr.getStoreZoneId(ref.getDataStoreId(), ref.getRole())) { + snapshotStore = ref; + break; + } + } if (snapshotStore == null) { if (!retrieveAnySnapshotFromVolume) { return null; } - - snapshotStore = snapshotStoreDao.findByVolume(snapshotId, snapshot.getVolumeId(), role); + snapshotStores = snapshotStoreDao.findByVolume(snapshotId, snapshot.getVolumeId(), role); + for (SnapshotDataStoreVO ref : snapshotStores) { + if (zoneId == storeMgr.getStoreZoneId(ref.getDataStoreId(), ref.getRole())); { + snapshotStore = ref; + break; + } + } if (snapshotStore == null) { return null; } diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotObject.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotObject.java index 2e45bee94b4f..6cf68f64fd92 100644 --- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotObject.java +++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotObject.java @@ -26,6 +26,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.DataObjectInStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; @@ -64,6 +65,7 @@ public class SnapshotObject implements SnapshotInfo { private DataStore store; private Object payload; private Boolean fullBackup; + private String url; @Inject protected SnapshotDao snapshotDao; @Inject @@ -80,8 +82,12 @@ public class SnapshotObject implements SnapshotInfo { SnapshotDataStoreDao snapshotStoreDao; @Inject StorageStrategyFactory storageStrategyFactory; + @Inject + DataStoreManager dataStoreManager; private String installPath; // temporarily set installPath before passing to resource for entries with empty installPath for object store migration case + private Long zoneId = null; + public SnapshotObject() { } @@ -142,7 +148,7 @@ public List getChildren() { List children = new ArrayList<>(); if (vos != null) { for (SnapshotDataStoreVO vo : vos) { - SnapshotInfo info = snapshotFactory.getSnapshot(vo.getSnapshotId(), DataStoreRole.Image); + SnapshotInfo info = snapshotFactory.getSnapshot(vo.getSnapshotId(), vo.getDataStoreId(), DataStoreRole.Image); if (info != null) { children.add(info); } @@ -164,7 +170,7 @@ public boolean isRevertable() { @Override public long getPhysicalSize() { long physicalSize = 0; - SnapshotDataStoreVO snapshotStore = snapshotStoreDao.findBySnapshot(snapshot.getId(), DataStoreRole.Image); + SnapshotDataStoreVO snapshotStore = snapshotStoreDao.findByStoreSnapshot(DataStoreRole.Image, store.getId(), snapshot.getId()); if (snapshotStore != null) { physicalSize = snapshotStore.getPhysicalSize(); } @@ -194,9 +200,16 @@ public long getId() { @Override public String getUri() { + if (url != null) { + return url; + } return snapshot.getUuid(); } + public void setUrl(String url) { + this.url = url; + } + @Override public DataStore getDataStore() { return store; @@ -309,7 +322,10 @@ public long getDomainId() { @Override public Long getDataCenterId() { - return snapshot.getDataCenterId(); + if (zoneId == null) { + zoneId = dataStoreManager.getStoreZoneId(store.getId(), store.getRole()); + } + return zoneId; } public void processEvent(Snapshot.Event event) throws NoTransitionException { diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java index 0c65eb045336..4268cf6446fa 100644 --- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java +++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java @@ -25,8 +25,11 @@ import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult; import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult; import org.apache.cloudstack.engine.subsystem.api.storage.DataMotionService; +import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; +import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; +import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector; import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine.Event; import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStore; @@ -42,16 +45,25 @@ import org.apache.cloudstack.framework.async.AsyncCallbackDispatcher; import org.apache.cloudstack.framework.async.AsyncCompletionCallback; import org.apache.cloudstack.framework.async.AsyncRpcContext; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.framework.jobs.AsyncJob; import org.apache.cloudstack.storage.command.CommandResult; import org.apache.cloudstack.storage.command.CopyCmdAnswer; +import org.apache.cloudstack.storage.command.QuerySnapshotZoneCopyAnswer; +import org.apache.cloudstack.storage.command.QuerySnapshotZoneCopyCommand; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; +import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity; +import org.apache.cloudstack.storage.to.SnapshotObjectTO; import org.apache.log4j.Logger; -import com.cloud.storage.CreateSnapshotPayload; +import com.cloud.agent.api.Answer; +import com.cloud.configuration.Config; +import com.cloud.dc.DataCenter; import com.cloud.event.EventTypes; import com.cloud.event.UsageEventUtils; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.storage.CreateSnapshotPayload; import com.cloud.storage.DataStoreRole; import com.cloud.storage.Snapshot; import com.cloud.storage.SnapshotVO; @@ -82,6 +94,10 @@ public class SnapshotServiceImpl implements SnapshotService { private SnapshotDetailsDao _snapshotDetailsDao; @Inject VolumeDataFactory volFactory; + @Inject + EndPointSelector epSelector; + @Inject + ConfigurationDao _configDao; static private class CreateSnapshotContext extends AsyncRpcContext { final SnapshotInfo snapshot; @@ -120,6 +136,20 @@ public CopySnapshotContext(AsyncCompletionCallback callback, SnapshotInfo src } + static private class PrepareCopySnapshotContext extends AsyncRpcContext { + final SnapshotInfo snapshot; + final String copyUrlBase; + final AsyncCallFuture future; + + public PrepareCopySnapshotContext(AsyncCompletionCallback callback, SnapshotInfo snapshot, String copyUrlBase, AsyncCallFuture future) { + super(callback); + this.snapshot = snapshot; + this.copyUrlBase = copyUrlBase; + this.future = future; + } + + } + static private class RevertSnapshotContext extends AsyncRpcContext { final SnapshotInfo snapshot; final AsyncCallFuture future; @@ -132,6 +162,30 @@ public RevertSnapshotContext(AsyncCompletionCallback callback, SnapshotInfo s } + private String generateCopyUrlBase(String hostname, String dir) { + String scheme = "http"; + boolean _sslCopy = false; + String sslCfg = _configDao.getValue(Config.SecStorageEncryptCopy.toString()); + String _ssvmUrlDomain = _configDao.getValue("secstorage.ssl.cert.domain"); + if (sslCfg != null) { + _sslCopy = Boolean.parseBoolean(sslCfg); + } + if(_sslCopy && (_ssvmUrlDomain == null || _ssvmUrlDomain.isEmpty())){ + s_logger.warn("Empty secondary storage url domain, ignoring SSL"); + _sslCopy = false; + } + if (_sslCopy) { + if(_ssvmUrlDomain.startsWith("*")) { + hostname = hostname.replace(".", "-"); + hostname = hostname + _ssvmUrlDomain.substring(1); + } else { + hostname = _ssvmUrlDomain; + } + scheme = "https"; + } + return scheme + "://" + hostname + "/copy/SecStorage/" + dir; + } + protected Void createSnapshotAsyncCallback(AsyncCallbackDispatcher callback, CreateSnapshotContext context) { CreateCmdResult result = callback.getResult(); SnapshotObject snapshot = (SnapshotObject)context.snapshot; @@ -251,7 +305,13 @@ private DataStore findSnapshotImageStore(SnapshotInfo snapshot) { // find the image store where the parent snapshot backup is located SnapshotDataStoreVO parentSnapshotOnBackupStore = null; if (parentSnapshot != null) { - parentSnapshotOnBackupStore = _snapshotStoreDao.findBySnapshot(parentSnapshot.getId(), DataStoreRole.Image); + List snaps = _snapshotStoreDao.listReadyBySnapshot(snapshot.getId(), DataStoreRole.Image); + for (SnapshotDataStoreVO ref : snaps) { + if (snapshot.getDataCenterId() != null && snapshot.getDataCenterId().equals(dataStoreMgr.getStoreZoneId(ref.getDataStoreId(), ref.getRole()))) { + parentSnapshotOnBackupStore = ref; + break; + } + } } if (parentSnapshotOnBackupStore == null) { return dataStoreMgr.getImageStoreWithFreeCapacity(snapshot.getDataCenterId()); @@ -356,6 +416,49 @@ protected Void copySnapshotAsyncCallback(AsyncCallbackDispatcher callback, CopySnapshotContext context) { + CreateCmdResult result = callback.getResult(); + SnapshotInfo destSnapshot = context.destSnapshot; + AsyncCallFuture future = context.future; + SnapshotResult snapResult = new SnapshotResult(destSnapshot, result.getAnswer()); + if (result.isFailed()) { + snapResult.setResult(result.getResult()); + destSnapshot.processEvent(Event.OperationFailed); + future.complete(snapResult); + return null; + } + try { + Answer answer = result.getAnswer(); + destSnapshot.processEvent(Event.OperationSuccessed); + snapResult = new SnapshotResult(_snapshotFactory.getSnapshot(destSnapshot.getId(), destSnapshot.getDataStore()), answer); + future.complete(snapResult); + } catch (Exception e) { + s_logger.debug("Failed to update snapshot state", e); + snapResult.setResult(e.toString()); + future.complete(snapResult); + } + return null; + } + + protected Void prepareCopySnapshotZoneAsyncCallback(AsyncCallbackDispatcher callback, PrepareCopySnapshotContext context) { + QuerySnapshotZoneCopyAnswer answer = callback.getResult(); + if (answer == null || !answer.getResult()) { + CreateCmdResult result = new CreateCmdResult(null, answer); + result.setResult(answer != null ? answer.getDetails() : "Unsupported answer"); + context.future.complete(result); + return null; + } + List files = answer.getFiles(); + final String copyUrlBase = context.copyUrlBase; + StringBuilder url = new StringBuilder(); + for (String file : files) { + url.append(copyUrlBase).append("/").append(file).append("\n"); + } + CreateCmdResult result = new CreateCmdResult(url.toString().trim(), answer); + context.future.complete(result); + return null; + } + protected Void deleteSnapshotCallback(AsyncCallbackDispatcher callback, DeleteSnapshotContext context) { CommandResult result = callback.getResult(); @@ -432,7 +535,7 @@ public boolean deleteSnapshot(SnapshotInfo snapInfo) { @Override public boolean revertSnapshot(SnapshotInfo snapshot) { PrimaryDataStore store = null; - SnapshotInfo snapshotOnPrimaryStore = _snapshotFactory.getSnapshot(snapshot.getId(), DataStoreRole.Primary); + SnapshotInfo snapshotOnPrimaryStore = _snapshotFactory.getSnapshotOnPrimaryStore(snapshot.getId()); if (snapshotOnPrimaryStore == null) { s_logger.warn("Cannot find an entry for snapshot " + snapshot.getId() + " on primary storage pools, searching with volume's primary storage pool"); VolumeInfo volumeInfo = volFactory.getVolume(snapshot.getVolumeId(), DataStoreRole.Primary); @@ -608,4 +711,56 @@ public void doInTransactionWithoutResult(TransactionStatus status) { } + @Override + public AsyncCallFuture copySnapshot(SnapshotInfo snapshot, String copyUrl, DataStore store) throws ResourceUnavailableException { + SnapshotObject snapshotForCopy = (SnapshotObject)_snapshotFactory.getSnapshot(snapshot, store); + snapshotForCopy.setUrl(copyUrl); + + if (s_logger.isDebugEnabled()) { + s_logger.debug("Mark snapshot_store_ref entry as Creating"); + } + AsyncCallFuture future = new AsyncCallFuture(); + DataObject snapshotOnStore = store.create(snapshotForCopy); + ((SnapshotObject)snapshotOnStore).setUrl(copyUrl); + snapshotOnStore.processEvent(Event.CreateOnlyRequested); + + if (s_logger.isDebugEnabled()) { + s_logger.debug("Invoke datastore driver createAsync to create snapshot on destination store"); + } + try { + CopySnapshotContext context = new CopySnapshotContext<>(null, (SnapshotObject)snapshotOnStore, snapshotForCopy, future); + AsyncCallbackDispatcher caller = AsyncCallbackDispatcher.create(this); + caller.setCallback(caller.getTarget().copySnapshotZoneAsyncCallback(null, null)).setContext(context); + store.getDriver().createAsync(store, snapshotOnStore, caller); + } catch (CloudRuntimeException ex) { + // clean up already persisted snapshot_store_ref entry + SnapshotDataStoreVO snapshotStoreVO = _snapshotStoreDao.findByStoreSnapshot(DataStoreRole.Image, store.getId(), snapshot.getId()); + if (snapshotStoreVO != null) { + snapshotForCopy.processEvent(ObjectInDataStoreStateMachine.Event.OperationFailed); + } + SnapshotResult res = new SnapshotResult((SnapshotObject)snapshotOnStore, null); + res.setResult(ex.getMessage()); + future.complete(res); + } + return future; + } + + @Override + public AsyncCallFuture queryCopySnapshot(SnapshotInfo snapshot) throws ResourceUnavailableException { + AsyncCallFuture future = new AsyncCallFuture<>(); + EndPoint ep = epSelector.select(snapshot); + if (ep == null) { + s_logger.error(String.format("Failed to find endpoint for generating copy URL for snapshot %d with store %d", snapshot.getId(), snapshot.getDataStore().getId())); + throw new ResourceUnavailableException("No secondary VM in running state in source snapshot zone", DataCenter.class, snapshot.getDataCenterId()); + } + DataStore store = snapshot.getDataStore(); + String copyUrlBase = generateCopyUrlBase(ep.getPublicAddr(), ((ImageStoreEntity)store).getMountPoint()); + PrepareCopySnapshotContext context = new PrepareCopySnapshotContext<>(null, snapshot, copyUrlBase, future); + AsyncCallbackDispatcher caller = AsyncCallbackDispatcher.create(this); + caller.setCallback(caller.getTarget().prepareCopySnapshotZoneAsyncCallback(null, null)).setContext(context); + caller.setContext(context); + QuerySnapshotZoneCopyCommand cmd = new QuerySnapshotZoneCopyCommand((SnapshotObjectTO)(snapshot.getTO())); + ep.sendMessageAsync(cmd, caller); + return future; + } } diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/StorageSystemSnapshotStrategy.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/StorageSystemSnapshotStrategy.java index 6401f8a8e1c9..dabb8d17702d 100644 --- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/StorageSystemSnapshotStrategy.java +++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/StorageSystemSnapshotStrategy.java @@ -44,6 +44,7 @@ import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -150,7 +151,7 @@ public SnapshotInfo backupSnapshot(SnapshotInfo snapshotInfo) { } @Override - public boolean deleteSnapshot(Long snapshotId) { + public boolean deleteSnapshot(Long snapshotId, Long zoneId) { Preconditions.checkArgument(snapshotId != null, "'snapshotId' cannot be 'null'."); SnapshotVO snapshotVO = snapshotDao.findById(snapshotId); @@ -181,7 +182,7 @@ public boolean deleteSnapshot(Long snapshotId) { */ @ActionEvent(eventType = EventTypes.EVENT_SNAPSHOT_OFF_PRIMARY, eventDescription = "deleting snapshot", async = true) private boolean cleanupSnapshotOnPrimaryStore(long snapshotId) { - SnapshotObject snapshotObj = (SnapshotObject)snapshotDataFactory.getSnapshot(snapshotId, DataStoreRole.Primary); + SnapshotObject snapshotObj = (SnapshotObject)snapshotDataFactory.getSnapshotOnPrimaryStore(snapshotId); if (snapshotObj == null) { s_logger.debug("Can't find snapshot; deleting it in DB"); @@ -293,7 +294,7 @@ public boolean revertSnapshot(SnapshotInfo snapshotInfo) { verifySnapshotType(snapshotInfo); - SnapshotDataStoreVO snapshotStore = snapshotStoreDao.findBySnapshot(snapshotInfo.getId(), DataStoreRole.Primary); + SnapshotDataStoreVO snapshotStore = snapshotStoreDao.findOneBySnapshotAndDatastoreRole(snapshotInfo.getId(), DataStoreRole.Primary); if (snapshotStore != null) { long snapshotStoragePoolId = snapshotStore.getDataStoreId(); @@ -911,7 +912,7 @@ private boolean usingBackendSnapshotFor(long snapshotId) { } @Override - public StrategyPriority canHandle(Snapshot snapshot, SnapshotOperation op) { + public StrategyPriority canHandle(Snapshot snapshot, Long zoneId, SnapshotOperation op) { Snapshot.LocationType locationType = snapshot.getLocationType(); // If the snapshot exists on Secondary Storage, we can't delete it. @@ -920,20 +921,26 @@ public StrategyPriority canHandle(Snapshot snapshot, SnapshotOperation op) { return StrategyPriority.CANT_HANDLE; } - SnapshotDataStoreVO snapshotStore = snapshotStoreDao.findBySnapshot(snapshot.getId(), DataStoreRole.Image); + List snapshotOnImageStores = snapshotStoreDao.listReadyBySnapshot(snapshot.getId(), DataStoreRole.Image); // If the snapshot exists on Secondary Storage, we can't delete it. - if (snapshotStore != null) { + if (CollectionUtils.isNotEmpty(snapshotOnImageStores)) { return StrategyPriority.CANT_HANDLE; } - snapshotStore = snapshotStoreDao.findBySnapshot(snapshot.getId(), DataStoreRole.Primary); + SnapshotDataStoreVO snapshotStore = snapshotStoreDao.findOneBySnapshotAndDatastoreRole(snapshot.getId(), DataStoreRole.Primary); if (snapshotStore == null) { return StrategyPriority.CANT_HANDLE; } long snapshotStoragePoolId = snapshotStore.getDataStoreId(); + if (zoneId != null) { // If zoneId is present, then it should be same as the zoneId of primary store + StoragePoolVO storagePoolVO = storagePoolDao.findById(snapshotStoragePoolId); + if (!zoneId.equals(storagePoolVO.getDataCenterId())) { + return StrategyPriority.CANT_HANDLE; + } + } boolean storageSystemSupportsCapability = storageSystemSupportsCapability(snapshotStoragePoolId, DataStoreCapabilities.STORAGE_SYSTEM_SNAPSHOT.toString()); @@ -953,7 +960,7 @@ public StrategyPriority canHandle(Snapshot snapshot, SnapshotOperation op) { boolean acceptableFormat = isAcceptableRevertFormat(volumeVO); if (acceptableFormat) { - SnapshotDataStoreVO snapshotStore = snapshotStoreDao.findBySnapshot(snapshot.getId(), DataStoreRole.Primary); + SnapshotDataStoreVO snapshotStore = snapshotStoreDao.findOneBySnapshotAndDatastoreRole(snapshot.getId(), DataStoreRole.Primary); boolean usingBackendSnapshot = usingBackendSnapshotFor(snapshot.getId()); diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/StorageVMSnapshotStrategy.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/StorageVMSnapshotStrategy.java index 709661f75ad5..f5d70817333c 100644 --- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/StorageVMSnapshotStrategy.java +++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/StorageVMSnapshotStrategy.java @@ -55,7 +55,6 @@ import com.cloud.exception.OperationTimedoutException; import com.cloud.hypervisor.Hypervisor; import com.cloud.storage.CreateSnapshotPayload; -import com.cloud.storage.DataStoreRole; import com.cloud.storage.GuestOSVO; import com.cloud.storage.Snapshot; import com.cloud.storage.SnapshotVO; @@ -390,7 +389,7 @@ protected void deleteSnapshotByStrategy(SnapshotVO snapshot) { //The snapshot could not be deleted separately, that's why we set snapshot state to BackedUp for operation delete VM snapshots and rollback SnapshotStrategy strategy = storageStrategyFactory.getSnapshotStrategy(snapshot, SnapshotOperation.DELETE); if (strategy != null) { - boolean snapshotForDelete = strategy.deleteSnapshot(snapshot.getId()); + boolean snapshotForDelete = strategy.deleteSnapshot(snapshot.getId(), null); if (!snapshotForDelete) { throw new CloudRuntimeException("Failed to delete snapshot"); } @@ -415,7 +414,7 @@ protected void deleteDiskSnapshot(VMSnapshot vmSnapshot) { protected void revertDiskSnapshot(VMSnapshot vmSnapshot) { List listSnapshots = vmSnapshotDetailsDao.findDetails(vmSnapshot.getId(), STORAGE_SNAPSHOT); for (VMSnapshotDetailsVO vmSnapshotDetailsVO : listSnapshots) { - SnapshotInfo sInfo = snapshotDataFactory.getSnapshot(Long.parseLong(vmSnapshotDetailsVO.getValue()), DataStoreRole.Primary); + SnapshotInfo sInfo = snapshotDataFactory.getSnapshotOnPrimaryStore(Long.parseLong(vmSnapshotDetailsVO.getValue())); SnapshotStrategy snapshotStrategy = storageStrategyFactory.getSnapshotStrategy(sInfo, SnapshotOperation.REVERT); if (snapshotStrategy == null) { throw new CloudRuntimeException(String.format("Could not find strategy for snapshot uuid [%s]", sInfo.getId())); diff --git a/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/CephSnapshotStrategyTest.java b/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/CephSnapshotStrategyTest.java index dcc6acf983fe..b33f57c685c5 100644 --- a/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/CephSnapshotStrategyTest.java +++ b/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/CephSnapshotStrategyTest.java @@ -81,10 +81,10 @@ private void configureAndVerifyCanHandle(Date removed, boolean isSnapshotStoredO VolumeVO volumeVO = Mockito.mock(VolumeVO.class); Mockito.when(volumeVO.getRemoved()).thenReturn(removed); Mockito.when(volumeDao.findByIdIncludingRemoved(Mockito.anyLong())).thenReturn(volumeVO); - Mockito.lenient().doReturn(isSnapshotStoredOnRbdStoragePool).when(cephSnapshotStrategy).isSnapshotStoredOnRbdStoragePool(Mockito.any()); + Mockito.lenient().doReturn(isSnapshotStoredOnRbdStoragePool).when(cephSnapshotStrategy).isSnapshotStoredOnRbdStoragePoolAndOperationForSameZone(Mockito.any(), Mockito.any()); for (int i = 0; i < snapshotOps.length - 1; i++) { - StrategyPriority strategyPriority = cephSnapshotStrategy.canHandle(snapshot, snapshotOps[i]); + StrategyPriority strategyPriority = cephSnapshotStrategy.canHandle(snapshot, null, snapshotOps[i]); if (snapshotOps[i] == SnapshotOperation.REVERT && isSnapshotStoredOnRbdStoragePool) { Assert.assertEquals(StrategyPriority.HIGHEST, strategyPriority); } else { diff --git a/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/DefaultSnapshotStrategyTest.java b/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/DefaultSnapshotStrategyTest.java index a092f8f108eb..09e5c85b770f 100644 --- a/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/DefaultSnapshotStrategyTest.java +++ b/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/DefaultSnapshotStrategyTest.java @@ -18,17 +18,16 @@ package org.apache.cloudstack.storage.snapshot; import java.util.ArrayList; -import java.util.LinkedHashMap; import java.util.List; -import java.util.Map; -import com.cloud.storage.VolumeDetailVO; -import com.cloud.storage.dao.VolumeDetailsDao; -import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; +import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotService; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -41,7 +40,13 @@ import com.cloud.storage.DataStoreRole; import com.cloud.storage.Snapshot; import com.cloud.storage.SnapshotVO; +import com.cloud.storage.Storage; +import com.cloud.storage.VolumeDetailVO; +import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.SnapshotDao; +import com.cloud.storage.dao.SnapshotZoneDao; +import com.cloud.storage.dao.VolumeDetailsDao; +import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.fsm.NoTransitionException; @RunWith(MockitoJUnitRunner.class) @@ -74,27 +79,31 @@ public class DefaultSnapshotStrategyTest { @Mock SnapshotService snapshotServiceMock; - Map mapStringSnapshotInfoInstance = new LinkedHashMap<>(); + @Mock + SnapshotZoneDao snapshotZoneDaoMock; + + @Mock + SnapshotDataStoreDao snapshotDataStoreDao; + + @Mock + DataStoreManager dataStoreManager; + + List mockSnapshotInfos = new ArrayList<>(); @Before public void setup() { - mapStringSnapshotInfoInstance.put("secondary storage", snapshotInfo1Mock); - mapStringSnapshotInfoInstance.put("primary storage", snapshotInfo1Mock); + mockSnapshotInfos.add(snapshotInfo1Mock); + mockSnapshotInfos.add(snapshotInfo2Mock); } @Test public void validateRetrieveSnapshotEntries() { Long snapshotId = 1l; - Mockito.doReturn(snapshotInfo1Mock, snapshotInfo2Mock).when(snapshotDataFactoryMock).getSnapshot(Mockito.anyLong(), Mockito.any(DataStoreRole.class), Mockito.anyBoolean()); - Map result = defaultSnapshotStrategySpy.retrieveSnapshotEntries(snapshotId); - - Mockito.verify(snapshotDataFactoryMock).getSnapshot(snapshotId, DataStoreRole.Image, false); - Mockito.verify(snapshotDataFactoryMock).getSnapshot(snapshotId, DataStoreRole.Primary, false); + Mockito.doReturn(mockSnapshotInfos).when(snapshotDataFactoryMock).getSnapshots(Mockito.anyLong(), Mockito.any()); + List result = defaultSnapshotStrategySpy.retrieveSnapshotEntries(snapshotId, null); - Assert.assertTrue(result.containsKey("secondary storage")); - Assert.assertTrue(result.containsKey("primary storage")); - Assert.assertEquals(snapshotInfo1Mock, result.get("secondary storage")); - Assert.assertEquals(snapshotInfo2Mock, result.get("primary storage")); + Assert.assertTrue(result.contains(snapshotInfo1Mock)); + Assert.assertTrue(result.contains(snapshotInfo2Mock)); } @Test @@ -107,38 +116,29 @@ public void validateUpdateSnapshotToDestroyed() { @Test public void validateDestroySnapshotEntriesAndFilesFailToDeleteReturnsFalse() { - Mockito.doReturn(false).when(defaultSnapshotStrategySpy).deleteSnapshotInfos(Mockito.any()); - Assert.assertFalse(defaultSnapshotStrategySpy.destroySnapshotEntriesAndFiles(snapshotVoMock)); + Mockito.doReturn(false).when(defaultSnapshotStrategySpy).deleteSnapshotInfos(Mockito.any(), Mockito.any()); + Assert.assertFalse(defaultSnapshotStrategySpy.destroySnapshotEntriesAndFiles(snapshotVoMock, null)); } @Test public void validateDestroySnapshotEntriesAndFilesDeletesSuccessfullyReturnsTrue() { - Mockito.doReturn(true).when(defaultSnapshotStrategySpy).deleteSnapshotInfos(Mockito.any()); - Assert.assertTrue(defaultSnapshotStrategySpy.destroySnapshotEntriesAndFiles(snapshotVoMock)); + Mockito.doReturn(true).when(defaultSnapshotStrategySpy).deleteSnapshotInfos(Mockito.any(), Mockito.any()); + Mockito.doNothing().when(snapshotZoneDaoMock).removeSnapshotFromZones(Mockito.anyLong()); + Assert.assertTrue(defaultSnapshotStrategySpy.destroySnapshotEntriesAndFiles(snapshotVoMock, null)); } @Test public void validateDeleteSnapshotInfosFailToDeleteReturnsFalse() { - Mockito.doReturn(mapStringSnapshotInfoInstance).when(defaultSnapshotStrategySpy).retrieveSnapshotEntries(Mockito.anyLong()); - Mockito.doReturn(false).when(defaultSnapshotStrategySpy).deleteSnapshotInfo(Mockito.any(), Mockito.anyString(), Mockito.any()); - Assert.assertFalse(defaultSnapshotStrategySpy.deleteSnapshotInfos(snapshotVoMock)); + Mockito.doReturn(mockSnapshotInfos).when(defaultSnapshotStrategySpy).retrieveSnapshotEntries(Mockito.anyLong(), Mockito.any()); + Mockito.doReturn(false).when(defaultSnapshotStrategySpy).deleteSnapshotInfo(Mockito.any(), Mockito.any()); + Assert.assertFalse(defaultSnapshotStrategySpy.deleteSnapshotInfos(snapshotVoMock, null)); } @Test public void validateDeleteSnapshotInfosDeletesSuccessfullyReturnsTrue() { - Mockito.doReturn(mapStringSnapshotInfoInstance).when(defaultSnapshotStrategySpy).retrieveSnapshotEntries(Mockito.anyLong()); - Mockito.doReturn(true).when(defaultSnapshotStrategySpy).deleteSnapshotInfo(Mockito.any(), Mockito.anyString(), Mockito.any()); - Assert.assertTrue(defaultSnapshotStrategySpy.deleteSnapshotInfos(snapshotVoMock)); - } - - @Test - public void validateDeleteSnapshotInfoSnapshotInfoIsNullOnSecondaryStorageReturnsTrue() { - Assert.assertNull(defaultSnapshotStrategySpy.deleteSnapshotInfo(null, "secondary storage", snapshotVoMock)); - } - - @Test - public void validateDeleteSnapshotInfoSnapshotInfoIsNullOnPrimaryStorageReturnsFalse() { - Assert.assertTrue(defaultSnapshotStrategySpy.deleteSnapshotInfo(null, "primary storage", snapshotVoMock)); + Mockito.doReturn(mockSnapshotInfos).when(defaultSnapshotStrategySpy).retrieveSnapshotEntries(Mockito.anyLong(), Mockito.any()); + Mockito.doReturn(true).when(defaultSnapshotStrategySpy).deleteSnapshotInfo(Mockito.any(), Mockito.any()); + Assert.assertTrue(defaultSnapshotStrategySpy.deleteSnapshotInfos(snapshotVoMock, null)); } @Test @@ -147,8 +147,9 @@ public void deleteSnapshotInfoTestReturnTrueIfCanDeleteTheSnapshotOnPrimaryStora Mockito.doReturn(snapshotObjectMock).when(defaultSnapshotStrategySpy).castSnapshotInfoToSnapshotObject(snapshotInfo1Mock); Mockito.doNothing().when(snapshotObjectMock).processEvent(Mockito.any(Snapshot.Event.class)); Mockito.doReturn(true).when(snapshotServiceMock).deleteSnapshot(Mockito.any()); + Mockito.when(dataStoreMock.getRole()).thenReturn(DataStoreRole.Primary); - boolean result = defaultSnapshotStrategySpy.deleteSnapshotInfo(snapshotInfo1Mock, "primary storage", snapshotVoMock); + boolean result = defaultSnapshotStrategySpy.deleteSnapshotInfo(snapshotInfo1Mock, snapshotVoMock); Assert.assertTrue(result); } @@ -158,8 +159,9 @@ public void deleteSnapshotInfoTestReturnFalseIfCannotDeleteTheSnapshotOnPrimaryS Mockito.doReturn(snapshotObjectMock).when(defaultSnapshotStrategySpy).castSnapshotInfoToSnapshotObject(snapshotInfo1Mock); Mockito.doNothing().when(snapshotObjectMock).processEvent(Mockito.any(Snapshot.Event.class)); Mockito.doReturn(false).when(snapshotServiceMock).deleteSnapshot(Mockito.any()); + Mockito.when(dataStoreMock.getRole()).thenReturn(DataStoreRole.Primary); - boolean result = defaultSnapshotStrategySpy.deleteSnapshotInfo(snapshotInfo1Mock, "primary storage", snapshotVoMock); + boolean result = defaultSnapshotStrategySpy.deleteSnapshotInfo(snapshotInfo1Mock, snapshotVoMock); Assert.assertFalse(result); } @@ -169,8 +171,9 @@ public void deleteSnapshotInfoTestReturnFalseIfDeleteSnapshotOnPrimaryStorageThr Mockito.doReturn(snapshotObjectMock).when(defaultSnapshotStrategySpy).castSnapshotInfoToSnapshotObject(snapshotInfo1Mock); Mockito.doNothing().when(snapshotObjectMock).processEvent(Mockito.any(Snapshot.Event.class)); Mockito.doThrow(CloudRuntimeException.class).when(snapshotServiceMock).deleteSnapshot(Mockito.any()); + Mockito.when(dataStoreMock.getRole()).thenReturn(DataStoreRole.Primary); - boolean result = defaultSnapshotStrategySpy.deleteSnapshotInfo(snapshotInfo1Mock, "primary storage", snapshotVoMock); + boolean result = defaultSnapshotStrategySpy.deleteSnapshotInfo(snapshotInfo1Mock, snapshotVoMock); Assert.assertFalse(result); } @@ -181,8 +184,9 @@ public void deleteSnapshotInfoTestReturnTrueIfCanDeleteTheSnapshotChainForSecond Mockito.doNothing().when(defaultSnapshotStrategySpy).verifyIfTheSnapshotIsBeingUsedByAnyVolume(snapshotObjectMock); Mockito.doNothing().when(snapshotObjectMock).processEvent(Mockito.any(Snapshot.Event.class)); Mockito.doReturn(true).when(defaultSnapshotStrategySpy).deleteSnapshotChain(Mockito.any(), Mockito.anyString()); + Mockito.when(dataStoreMock.getRole()).thenReturn(DataStoreRole.Image); - boolean result = defaultSnapshotStrategySpy.deleteSnapshotInfo(snapshotInfo1Mock, "secondary storage", snapshotVoMock); + boolean result = defaultSnapshotStrategySpy.deleteSnapshotInfo(snapshotInfo1Mock, snapshotVoMock); Assert.assertTrue(result); } @@ -193,8 +197,9 @@ public void deleteSnapshotInfoTestReturnTrueIfCannotDeleteTheSnapshotChainForSec Mockito.doNothing().when(defaultSnapshotStrategySpy).verifyIfTheSnapshotIsBeingUsedByAnyVolume(snapshotObjectMock); Mockito.doNothing().when(snapshotObjectMock).processEvent(Mockito.any(Snapshot.Event.class)); Mockito.doReturn(false).when(defaultSnapshotStrategySpy).deleteSnapshotChain(Mockito.any(), Mockito.anyString()); + Mockito.when(dataStoreMock.getRole()).thenReturn(DataStoreRole.Image); - boolean result = defaultSnapshotStrategySpy.deleteSnapshotInfo(snapshotInfo1Mock, "secondary storage", snapshotVoMock); + boolean result = defaultSnapshotStrategySpy.deleteSnapshotInfo(snapshotInfo1Mock, snapshotVoMock); Assert.assertTrue(result); } @@ -203,8 +208,9 @@ public void validateDeleteSnapshotInfoSnapshotProcessSnapshotEventThrowsNoTransi Mockito.doReturn(dataStoreMock).when(snapshotInfo1Mock).getDataStore(); Mockito.doReturn(snapshotObjectMock).when(defaultSnapshotStrategySpy).castSnapshotInfoToSnapshotObject(snapshotInfo1Mock); Mockito.doThrow(NoTransitionException.class).when(snapshotObjectMock).processEvent(Mockito.any(Snapshot.Event.class)); + Mockito.when(dataStoreMock.getRole()).thenReturn(DataStoreRole.Image); - Assert.assertFalse(defaultSnapshotStrategySpy.deleteSnapshotInfo(snapshotInfo1Mock, "secondary storage", snapshotVoMock)); + Assert.assertFalse(defaultSnapshotStrategySpy.deleteSnapshotInfo(snapshotInfo1Mock, snapshotVoMock)); } @Test @@ -231,18 +237,97 @@ public void verifyIfTheSnapshotIsBeingUsedByAnyVolumeTestDetailsIsNotEmptyThrowC public void deleteSnapshotInPrimaryStorageTestReturnTrueIfDeleteReturnsTrue() throws NoTransitionException { Mockito.doReturn(true).when(snapshotServiceMock).deleteSnapshot(Mockito.any()); Mockito.doNothing().when(snapshotObjectMock).processEvent(Mockito.any(Snapshot.Event.class)); - Assert.assertTrue(defaultSnapshotStrategySpy.deleteSnapshotInPrimaryStorage(null, null, null, snapshotObjectMock)); + Assert.assertTrue(defaultSnapshotStrategySpy.deleteSnapshotInPrimaryStorage(null, null, null, snapshotObjectMock, true)); + } + + @Test + public void deleteSnapshotInPrimaryStorageTestReturnTrueIfDeleteNotLastRefReturnsTrue() throws NoTransitionException { + Mockito.doReturn(true).when(snapshotServiceMock).deleteSnapshot(Mockito.any()); + Assert.assertTrue(defaultSnapshotStrategySpy.deleteSnapshotInPrimaryStorage(null, null, null, snapshotObjectMock, false)); } @Test public void deleteSnapshotInPrimaryStorageTestReturnFalseIfDeleteReturnsFalse() throws NoTransitionException { Mockito.doReturn(false).when(snapshotServiceMock).deleteSnapshot(Mockito.any()); - Assert.assertFalse(defaultSnapshotStrategySpy.deleteSnapshotInPrimaryStorage(null, null, null, null)); + Assert.assertFalse(defaultSnapshotStrategySpy.deleteSnapshotInPrimaryStorage(null, null, null, null, true)); } @Test public void deleteSnapshotInPrimaryStorageTestReturnFalseIfDeleteThrowsException() throws NoTransitionException { Mockito.doThrow(CloudRuntimeException.class).when(snapshotServiceMock).deleteSnapshot(Mockito.any()); - Assert.assertFalse(defaultSnapshotStrategySpy.deleteSnapshotInPrimaryStorage(null, null, null, null)); + Assert.assertFalse(defaultSnapshotStrategySpy.deleteSnapshotInPrimaryStorage(null, null, null, null, true)); + } + + @Test + public void testGetSnapshotImageStoreRefNull() { + SnapshotDataStoreVO ref1 = Mockito.mock(SnapshotDataStoreVO.class); + Mockito.when(ref1.getDataStoreId()).thenReturn(1L); + Mockito.when(ref1.getRole()).thenReturn(DataStoreRole.Image); + Mockito.when(snapshotDataStoreDao.listReadyBySnapshot(Mockito.anyLong(), Mockito.any(DataStoreRole.class))).thenReturn(List.of(ref1)); + Mockito.when(dataStoreManager.getStoreZoneId(1L, DataStoreRole.Image)).thenReturn(2L); + Assert.assertNull(defaultSnapshotStrategySpy.getSnapshotImageStoreRef(1L, 1L)); + } + + @Test + public void testGetSnapshotImageStoreRefNotNull() { + SnapshotDataStoreVO ref1 = Mockito.mock(SnapshotDataStoreVO.class); + Mockito.when(ref1.getDataStoreId()).thenReturn(1L); + Mockito.when(ref1.getRole()).thenReturn(DataStoreRole.Image); + Mockito.when(snapshotDataStoreDao.listReadyBySnapshot(Mockito.anyLong(), Mockito.any(DataStoreRole.class))).thenReturn(List.of(ref1)); + Mockito.when(dataStoreManager.getStoreZoneId(1L, DataStoreRole.Image)).thenReturn(1L); + Assert.assertNotNull(defaultSnapshotStrategySpy.getSnapshotImageStoreRef(1L, 1L)); + } + + @Test + public void testIsSnapshotStoredOnSameZoneStoreForQCOW2VolumeNull() { + Assert.assertFalse(defaultSnapshotStrategySpy.isSnapshotStoredOnSameZoneStoreForQCOW2Volume(Mockito.mock(Snapshot.class), null)); + } + + @Test + public void testIsSnapshotStoredOnSameZoneStoreForQCOW2VolumeVHD() { + VolumeVO volumeVO = Mockito.mock((VolumeVO.class)); + Mockito.when(volumeVO.getFormat()).thenReturn(Storage.ImageFormat.VHD); + Assert.assertFalse(defaultSnapshotStrategySpy.isSnapshotStoredOnSameZoneStoreForQCOW2Volume(Mockito.mock(Snapshot.class), volumeVO)); + } + + private void prepareMocksForIsSnapshotStoredOnSameZoneStoreForQCOW2VolumeTest(Long matchingZoneId) { + SnapshotDataStoreVO ref1 = Mockito.mock(SnapshotDataStoreVO.class); + Mockito.when(ref1.getDataStoreId()).thenReturn(201L); + Mockito.when(ref1.getRole()).thenReturn(DataStoreRole.Image); + SnapshotDataStoreVO ref2 = Mockito.mock(SnapshotDataStoreVO.class); + Mockito.when(ref2.getDataStoreId()).thenReturn(202L); + Mockito.when(ref2.getRole()).thenReturn(DataStoreRole.Image); + SnapshotDataStoreVO ref3 = Mockito.mock(SnapshotDataStoreVO.class); + Mockito.when(ref3.getDataStoreId()).thenReturn(203L); + Mockito.when(ref3.getRole()).thenReturn(DataStoreRole.Image); + Mockito.when(snapshotDataStoreDao.listBySnapshotIdAndState(1L, ObjectInDataStoreStateMachine.State.Ready)).thenReturn(List.of(ref1, ref2, ref3)); + Mockito.when(dataStoreManager.getStoreZoneId(201L, DataStoreRole.Image)).thenReturn(111L); + Mockito.when(dataStoreManager.getStoreZoneId(202L, DataStoreRole.Image)).thenReturn(matchingZoneId != null ? matchingZoneId : 112L); + Mockito.when(dataStoreManager.getStoreZoneId(203L, DataStoreRole.Image)).thenReturn(113L); + + } + + @Test + public void testIsSnapshotStoredOnSameZoneStoreForQCOW2VolumeNoRef() { + Snapshot snapshot = Mockito.mock((Snapshot.class)); + Mockito.when(snapshot.getId()).thenReturn(1L); + VolumeVO volumeVO = Mockito.mock((VolumeVO.class)); + Mockito.when(volumeVO.getFormat()).thenReturn(Storage.ImageFormat.QCOW2); + Mockito.when(snapshotDataStoreDao.listBySnapshotIdAndState(1L, ObjectInDataStoreStateMachine.State.Ready)).thenReturn(new ArrayList<>()); + Assert.assertFalse(defaultSnapshotStrategySpy.isSnapshotStoredOnSameZoneStoreForQCOW2Volume(snapshot, volumeVO)); + + prepareMocksForIsSnapshotStoredOnSameZoneStoreForQCOW2VolumeTest(null); + Assert.assertFalse(defaultSnapshotStrategySpy.isSnapshotStoredOnSameZoneStoreForQCOW2Volume(snapshot, volumeVO)); + } + + @Test + public void testIsSnapshotStoredOnSameZoneStoreForQCOW2VolumeHasRef() { + Snapshot snapshot = Mockito.mock((Snapshot.class)); + Mockito.when(snapshot.getId()).thenReturn(1L); + VolumeVO volumeVO = Mockito.mock((VolumeVO.class)); + Mockito.when(volumeVO.getFormat()).thenReturn(Storage.ImageFormat.QCOW2); + Mockito.when(volumeVO.getDataCenterId()).thenReturn(100L); + prepareMocksForIsSnapshotStoredOnSameZoneStoreForQCOW2VolumeTest(100L); + Assert.assertTrue(defaultSnapshotStrategySpy.isSnapshotStoredOnSameZoneStoreForQCOW2Volume(snapshot, volumeVO)); } } diff --git a/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/SnapshotDataFactoryImplTest.java b/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/SnapshotDataFactoryImplTest.java index 520dbfa3c261..94e248149ea0 100644 --- a/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/SnapshotDataFactoryImplTest.java +++ b/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/SnapshotDataFactoryImplTest.java @@ -62,7 +62,7 @@ public class SnapshotDataFactoryImplTest { public void getSnapshotsByVolumeAndDataStoreTestNoSnapshotDataStoreVOFound() { Mockito.doReturn(new ArrayList<>()).when(snapshotStoreDaoMock).listAllByVolumeAndDataStore(volumeMockId, DataStoreRole.Primary); - List snapshots = snapshotDataFactoryImpl.getSnapshots(volumeMockId, DataStoreRole.Primary); + List snapshots = snapshotDataFactoryImpl.getSnapshotsForVolumeAndStoreRole(volumeMockId, DataStoreRole.Primary); Assert.assertTrue(snapshots.isEmpty()); } @@ -91,7 +91,7 @@ public void getSnapshotsByVolumeAndDataStoreTest() { Mockito.doReturn(dataStoreMock).when(dataStoreManagerMock).getDataStore(dataStoreId, dataStoreRole); Mockito.doReturn(snapshotVoMock).when(snapshotDaoMock).findById(snapshotId); - List snapshots = snapshotDataFactoryImpl.getSnapshots(volumeMockId, dataStoreRole); + List snapshots = snapshotDataFactoryImpl.getSnapshotsForVolumeAndStoreRole(volumeMockId, dataStoreRole); Assert.assertEquals(1, snapshots.size()); diff --git a/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImplTest.java b/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImplTest.java index ec2ab8a722b5..917fb2d9c750 100644 --- a/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImplTest.java +++ b/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImplTest.java @@ -18,7 +18,6 @@ */ package org.apache.cloudstack.storage.snapshot; -import com.cloud.storage.DataStoreRole; import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStore; import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreDriver; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory; @@ -41,6 +40,8 @@ import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.support.AnnotationConfigContextLoader; +import com.cloud.storage.DataStoreRole; + @RunWith(MockitoJUnitRunner.class) @ContextConfiguration(loader = AnnotationConfigContextLoader.class) public class SnapshotServiceImplTest { @@ -65,7 +66,7 @@ public void testRevertSnapshotWithNoPrimaryStorageEntry() throws Exception { Mockito.when(snapshot.getId()).thenReturn(1L); Mockito.when(snapshot.getVolumeId()).thenReturn(1L); - Mockito.when(_snapshotFactory.getSnapshot(1L, DataStoreRole.Primary)).thenReturn(null); + Mockito.when(_snapshotFactory.getSnapshotOnPrimaryStore(1L)).thenReturn(null); Mockito.when(volFactory.getVolume(1L, DataStoreRole.Primary)).thenReturn(volumeInfo); PrimaryDataStore store = Mockito.mock(PrimaryDataStore.class); @@ -82,5 +83,4 @@ public void testRevertSnapshotWithNoPrimaryStorageEntry() throws Exception { Assert.assertTrue(snapshotService.revertSnapshot(snapshot)); } } - } diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/DataStoreManagerImpl.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/DataStoreManagerImpl.java index ff6c4fb5c6a7..cd525ae0ef73 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/DataStoreManagerImpl.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/DataStoreManagerImpl.java @@ -170,4 +170,16 @@ public void setPrimaryStoreMgr(PrimaryDataStoreProviderManager primaryStoreMgr) public void setImageDataStoreMgr(ImageStoreProviderManager imageDataStoreMgr) { this.imageDataStoreMgr = imageDataStoreMgr; } + + @Override + public Long getStoreZoneId(long storeId, DataStoreRole role) { + try { + if (role == DataStoreRole.Primary) { + return primaryStoreMgr.getPrimaryDataStoreZoneId(storeId); + } else { + return imageDataStoreMgr.getImageStoreZoneId(storeId); + } + } catch (CloudRuntimeException ignored) {} + return null; + } } diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManager.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManager.java index 48acecab6b86..e822201d909c 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManager.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManager.java @@ -40,6 +40,4 @@ public interface ObjectInDataStoreManager { DataObjectInStore findObject(long objId, DataObjectType type, long dataStoreId, DataStoreRole role, String deployAsIsConfiguration); DataObjectInStore findObject(DataObject obj, DataStore store); - - DataStore findStore(long objId, DataObjectType type, DataStoreRole role); } diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java index da97b22946e1..47ec9890da83 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java @@ -382,27 +382,4 @@ public DataObjectInStore findObject(long objId, DataObjectType type, long dataSt } - @Override - public DataStore findStore(long objId, DataObjectType type, DataStoreRole role) { - DataStore store = null; - if (role == DataStoreRole.Image) { - DataObjectInStore vo = null; - switch (type) { - case TEMPLATE: - vo = templateDataStoreDao.findByTemplate(objId, role); - break; - case SNAPSHOT: - vo = snapshotDataStoreDao.findBySnapshot(objId, role); - break; - case VOLUME: - vo = volumeDataStoreDao.findByVolume(objId); - break; - } - if (vo != null) { - store = this.storeMgr.getDataStore(vo.getDataStoreId(), role); - } - } - return store; - } - } diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/PrimaryDataStoreProviderManager.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/PrimaryDataStoreProviderManager.java index bb7911df811e..8c8919c54122 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/PrimaryDataStoreProviderManager.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/PrimaryDataStoreProviderManager.java @@ -30,4 +30,6 @@ public interface PrimaryDataStoreProviderManager { boolean registerDriver(String providerName, PrimaryDataStoreDriver driver); boolean registerHostListener(String providerName, HypervisorHostListener listener); + + public long getPrimaryDataStoreZoneId(long dataStoreId); } diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/helper/StorageStrategyFactoryImpl.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/helper/StorageStrategyFactoryImpl.java index 9dbaf13010a1..ec76bbb62beb 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/helper/StorageStrategyFactoryImpl.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/helper/StorageStrategyFactoryImpl.java @@ -66,10 +66,15 @@ public StrategyPriority canHandle(DataMotionStrategy strategy) { @Override public SnapshotStrategy getSnapshotStrategy(final Snapshot snapshot, final SnapshotOperation op) { + return getSnapshotStrategy(snapshot, null, op); + } + + @Override + public SnapshotStrategy getSnapshotStrategy(Snapshot snapshot, Long zoneId, SnapshotOperation op) { return bestMatch(snapshotStrategies, new CanHandle() { @Override public StrategyPriority canHandle(SnapshotStrategy strategy) { - return strategy.canHandle(snapshot, op); + return strategy.canHandle(snapshot, zoneId, op); } }); } diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java index 3ef9fbc4225e..369630a1a73e 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java @@ -32,17 +32,11 @@ import javax.inject.Inject; -import com.cloud.agent.api.to.NfsTO; -import com.cloud.agent.api.to.OVFInformationTO; -import com.cloud.storage.DataStoreRole; -import com.cloud.storage.Upload; -import org.apache.cloudstack.storage.image.deployasis.DeployAsIsHelper; -import org.apache.log4j.Logger; - import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult; import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult; import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo; @@ -53,11 +47,15 @@ import org.apache.cloudstack.storage.command.CommandResult; import org.apache.cloudstack.storage.command.CopyCommand; import org.apache.cloudstack.storage.command.DeleteCommand; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreDao; import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO; import org.apache.cloudstack.storage.endpoint.DefaultEndPointSelector; +import org.apache.cloudstack.storage.image.deployasis.DeployAsIsHelper; +import org.apache.log4j.Logger; import com.cloud.agent.AgentManager; import com.cloud.agent.api.Answer; @@ -68,6 +66,8 @@ import com.cloud.agent.api.to.DataObjectType; import com.cloud.agent.api.to.DataTO; import com.cloud.agent.api.to.DatadiskTO; +import com.cloud.agent.api.to.NfsTO; +import com.cloud.agent.api.to.OVFInformationTO; import com.cloud.alert.AlertManager; import com.cloud.configuration.Config; import com.cloud.exception.AgentUnavailableException; @@ -76,7 +76,8 @@ import com.cloud.host.dao.HostDao; import com.cloud.secstorage.CommandExecLogDao; import com.cloud.secstorage.CommandExecLogVO; -import com.cloud.storage.StorageManager; +import com.cloud.storage.DataStoreRole; +import com.cloud.storage.Upload; import com.cloud.storage.VMTemplateStorageResourceAssoc; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.VolumeVO; @@ -84,8 +85,6 @@ import com.cloud.storage.dao.VMTemplateZoneDao; import com.cloud.storage.dao.VolumeDao; import com.cloud.storage.download.DownloadMonitor; -import com.cloud.user.ResourceLimitService; -import com.cloud.user.dao.AccountDao; import com.cloud.utils.NumbersUtil; import com.cloud.utils.db.TransactionLegacy; import com.cloud.utils.exception.CloudRuntimeException; @@ -107,6 +106,8 @@ public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver { @Inject TemplateDataStoreDao _templateStoreDao; @Inject + SnapshotDataStoreDao snapshotDataStoreDao; + @Inject EndPointSelector _epSelector; @Inject ConfigurationDao configDao; @@ -117,21 +118,17 @@ public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver { @Inject DefaultEndPointSelector _defaultEpSelector; @Inject - AccountDao _accountDao; - @Inject - ResourceLimitService _resourceLimitMgr; - @Inject DeployAsIsHelper deployAsIsHelper; @Inject HostDao hostDao; @Inject CommandExecLogDao _cmdExecLogDao; @Inject - StorageManager storageMgr; - @Inject protected SecondaryStorageVmDao _secStorageVmDao; @Inject AgentManager agentMgr; + @Inject + DataStoreManager dataStoreManager; protected String _proxy = null; @@ -192,6 +189,12 @@ public void createAsync(DataStore dataStore, DataObject data, AsyncCompletionCal LOGGER.debug("Downloading volume to data store " + dataStore.getId()); } _downloadMonitor.downloadVolumeToStorage(data, caller); + } else if (data.getType() == DataObjectType.SNAPSHOT) { + caller.setCallback(caller.getTarget().createSnapshotAsyncCallback(null, null)); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Downloading volume to data store " + dataStore.getId()); + } + _downloadMonitor.downloadSnapshotToStorage(data, caller); } } @@ -313,6 +316,53 @@ protected Void createTemplateAsyncCallback(AsyncCallbackDispatcher callback, CreateContext context) { + DownloadAnswer answer = callback.getResult(); + DataObject obj = context.data; + DataStore store = obj.getDataStore(); + + SnapshotDataStoreVO snapshotStoreVO = snapshotDataStoreDao.findByStoreSnapshot(DataStoreRole.Image, store.getId(), obj.getId()); + if (snapshotStoreVO != null) { + if (VMTemplateStorageResourceAssoc.Status.DOWNLOADED.equals(snapshotStoreVO.getDownloadState())) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Snapshot is already in DOWNLOADED state, ignore further incoming DownloadAnswer"); + } + return null; + } + SnapshotDataStoreVO updateBuilder = snapshotDataStoreDao.createForUpdate(); + updateBuilder.setDownloadPercent(answer.getDownloadPct()); + updateBuilder.setDownloadState(answer.getDownloadStatus()); + updateBuilder.setLastUpdated(new Date()); + updateBuilder.setErrorString(answer.getErrorString()); + updateBuilder.setJobId(answer.getJobId()); + updateBuilder.setLocalDownloadPath(answer.getDownloadPath()); + updateBuilder.setInstallPath(answer.getInstallPath()); + updateBuilder.setSize(answer.getTemplateSize()); + updateBuilder.setPhysicalSize(answer.getTemplatePhySicalSize()); + snapshotDataStoreDao.update(snapshotStoreVO.getId(), updateBuilder); + } + + AsyncCompletionCallback caller = context.getParentCallback(); + + if (List.of(VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR, + VMTemplateStorageResourceAssoc.Status.ABANDONED, + VMTemplateStorageResourceAssoc.Status.UNKNOWN).contains(answer.getDownloadStatus())) { + CreateCmdResult result = new CreateCmdResult(null, null); + result.setSuccess(false); + result.setResult(answer.getErrorString()); + caller.complete(result); + String msg = "Failed to copy snapshot: " + obj.getUuid() + " with error: " + answer.getErrorString(); + Long zoneId = dataStoreManager.getStoreZoneId(store.getId(), store.getRole()); + _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_UPLOAD_FAILED, + zoneId, null, msg, msg); + LOGGER.error(msg); + } else if (answer.getDownloadStatus() == VMTemplateStorageResourceAssoc.Status.DOWNLOADED) { + CreateCmdResult result = new CreateCmdResult(null, null); + caller.complete(result); + } + return null; + } + @Override public void deleteAsync(DataStore dataStore, DataObject data, AsyncCompletionCallback callback) { CommandResult result = new CommandResult(); @@ -331,7 +381,7 @@ public void deleteAsync(DataStore dataStore, DataObject data, AsyncCompletionCal result.setResult(answer.getDetails()); } } catch (Exception ex) { - LOGGER.debug("Unable to destoy " + data.getType().toString() + ": " + data.getId(), ex); + LOGGER.debug("Unable to destroy " + data.getType().toString() + ": " + data.getId(), ex); result.setResult(ex.toString()); } callback.complete(result); diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/datastore/ImageStoreProviderManager.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/datastore/ImageStoreProviderManager.java index 7e2f720042ed..47e2ee383070 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/datastore/ImageStoreProviderManager.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/datastore/ImageStoreProviderManager.java @@ -80,4 +80,5 @@ public interface ImageStoreProviderManager { List listImageStoresWithFreeCapacity(List imageStores); List orderImageStoresOnFreeCapacity(List imageStores); + long getImageStoreZoneId(long dataStoreId); } diff --git a/engine/storage/src/test/java/org/apache/cloudstack/engine/subsystem/api/storage/StrategyPriorityTest.java b/engine/storage/src/test/java/org/apache/cloudstack/engine/subsystem/api/storage/StrategyPriorityTest.java index bddd5a218807..493ea089ef49 100644 --- a/engine/storage/src/test/java/org/apache/cloudstack/engine/subsystem/api/storage/StrategyPriorityTest.java +++ b/engine/storage/src/test/java/org/apache/cloudstack/engine/subsystem/api/storage/StrategyPriorityTest.java @@ -25,14 +25,17 @@ import java.util.List; import java.util.Map; -import org.junit.Test; - -import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotStrategy.SnapshotOperation; import org.apache.cloudstack.storage.helper.StorageStrategyFactoryImpl; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; import com.cloud.host.Host; import com.cloud.storage.Snapshot; + +@RunWith(MockitoJUnitRunner.class) public class StrategyPriorityTest { @Test @@ -42,31 +45,31 @@ public void testSortSnapshotStrategies() { SnapshotStrategy hyperStrategy = mock(SnapshotStrategy.class); SnapshotStrategy highestStrategy = mock(SnapshotStrategy.class); - doReturn(StrategyPriority.CANT_HANDLE).when(cantHandleStrategy).canHandle(any(Snapshot.class), any(SnapshotOperation.class)); - doReturn(StrategyPriority.DEFAULT).when(defaultStrategy).canHandle(any(Snapshot.class), any(SnapshotOperation.class)); - doReturn(StrategyPriority.HYPERVISOR).when(hyperStrategy).canHandle(any(Snapshot.class), any(SnapshotOperation.class)); - doReturn(StrategyPriority.HIGHEST).when(highestStrategy).canHandle(any(Snapshot.class), any(SnapshotOperation.class)); + doReturn(StrategyPriority.CANT_HANDLE).when(cantHandleStrategy).canHandle(any(Snapshot.class), Mockito.nullable(Long.class), any(SnapshotStrategy.SnapshotOperation.class)); + doReturn(StrategyPriority.DEFAULT).when(defaultStrategy).canHandle(any(Snapshot.class), Mockito.nullable(Long.class), any(SnapshotStrategy.SnapshotOperation.class)); + doReturn(StrategyPriority.HYPERVISOR).when(hyperStrategy).canHandle(any(Snapshot.class), Mockito.nullable(Long.class), any(SnapshotStrategy.SnapshotOperation.class)); + doReturn(StrategyPriority.HIGHEST).when(highestStrategy).canHandle(any(Snapshot.class), Mockito.nullable(Long.class), any(SnapshotStrategy.SnapshotOperation.class)); - List strategies = new ArrayList(5); + List strategies = new ArrayList<>(5); SnapshotStrategy strategy = null; StorageStrategyFactoryImpl factory = new StorageStrategyFactoryImpl(); factory.setSnapshotStrategies(strategies); strategies.add(cantHandleStrategy); - strategy = factory.getSnapshotStrategy(mock(Snapshot.class), SnapshotOperation.TAKE); + strategy = factory.getSnapshotStrategy(mock(Snapshot.class), SnapshotStrategy.SnapshotOperation.TAKE); assertEquals("A strategy was found when it shouldn't have been.", null, strategy); strategies.add(defaultStrategy); - strategy = factory.getSnapshotStrategy(mock(Snapshot.class), SnapshotOperation.TAKE); + strategy = factory.getSnapshotStrategy(mock(Snapshot.class), SnapshotStrategy.SnapshotOperation.TAKE); assertEquals("Default strategy was not picked.", defaultStrategy, strategy); strategies.add(hyperStrategy); - strategy = factory.getSnapshotStrategy(mock(Snapshot.class), SnapshotOperation.TAKE); + strategy = factory.getSnapshotStrategy(mock(Snapshot.class), SnapshotStrategy.SnapshotOperation.TAKE); assertEquals("Hypervisor strategy was not picked.", hyperStrategy, strategy); strategies.add(highestStrategy); - strategy = factory.getSnapshotStrategy(mock(Snapshot.class), SnapshotOperation.TAKE); + strategy = factory.getSnapshotStrategy(mock(Snapshot.class), SnapshotStrategy.SnapshotOperation.TAKE); assertEquals("Highest strategy was not picked.", highestStrategy, strategy); } diff --git a/engine/storage/src/test/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImplTest.java b/engine/storage/src/test/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImplTest.java index b3b9efcbf908..cea6ac29f6ef 100644 --- a/engine/storage/src/test/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImplTest.java +++ b/engine/storage/src/test/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImplTest.java @@ -19,12 +19,6 @@ package org.apache.cloudstack.storage.datastore.db; -import com.cloud.hypervisor.Hypervisor; -import com.cloud.storage.DataStoreRole; -import com.cloud.storage.SnapshotVO; -import com.cloud.storage.dao.SnapshotDao; -import com.cloud.utils.db.SearchBuilder; -import com.cloud.utils.db.SearchCriteria; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -33,6 +27,13 @@ import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; +import com.cloud.hypervisor.Hypervisor; +import com.cloud.storage.DataStoreRole; +import com.cloud.storage.SnapshotVO; +import com.cloud.storage.dao.SnapshotDao; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; + @RunWith(MockitoJUnitRunner.class) public class SnapshotDataStoreDaoImplTest { @@ -61,17 +62,15 @@ public void init(){ @Test public void validateExpungeReferenceBySnapshotIdAndDataStoreRoleNullReference(){ - Mockito.doReturn(searchCriteriaMock).when(snapshotDataStoreDaoImplSpy).createSearchCriteriaBySnapshotIdAndStoreRole(Mockito.anyLong(), Mockito.any()); - Mockito.doReturn(null).when(snapshotDataStoreDaoImplSpy).findOneBy(searchCriteriaMock); - Assert.assertTrue(snapshotDataStoreDaoImplSpy.expungeReferenceBySnapshotIdAndDataStoreRole(0, DataStoreRole.Image)); + Mockito.doReturn(null).when(snapshotDataStoreDaoImplSpy).findByStoreSnapshot(Mockito.any(), Mockito.anyLong(), Mockito.anyLong()); + Assert.assertTrue(snapshotDataStoreDaoImplSpy.expungeReferenceBySnapshotIdAndDataStoreRole(0, 1L, DataStoreRole.Image)); } @Test public void validateExpungeReferenceBySnapshotIdAndDataStoreRole(){ - Mockito.doReturn(searchCriteriaMock).when(snapshotDataStoreDaoImplSpy).createSearchCriteriaBySnapshotIdAndStoreRole(Mockito.anyLong(), Mockito.any()); - Mockito.doReturn(snapshotDataStoreVoMock).when(snapshotDataStoreDaoImplSpy).findOneBy(searchCriteriaMock); + Mockito.doReturn(snapshotDataStoreVoMock).when(snapshotDataStoreDaoImplSpy).findByStoreSnapshot(Mockito.any(), Mockito.anyLong(), Mockito.anyLong()); Mockito.doReturn(true).when(snapshotDataStoreDaoImplSpy).expunge(Mockito.anyLong()); - Assert.assertTrue(snapshotDataStoreDaoImplSpy.expungeReferenceBySnapshotIdAndDataStoreRole(0, DataStoreRole.Image)); + Assert.assertTrue(snapshotDataStoreDaoImplSpy.expungeReferenceBySnapshotIdAndDataStoreRole(0, 1L, DataStoreRole.Image)); } @Test @@ -112,33 +111,30 @@ public void isSnapshotChainingRequiredTestSnapshotIsNotNullReturnAccordingHyperv @Test public void expungeReferenceBySnapshotIdAndDataStoreRoleTestSnapshotDataStoreIsNullReturnTrue() { - Mockito.doReturn(searchCriteriaMock).when(snapshotDataStoreDaoImplSpy).createSearchCriteriaBySnapshotIdAndStoreRole(Mockito.anyLong(), Mockito.any()); - Mockito.doReturn(null).when(snapshotDataStoreDaoImplSpy).findOneBy(Mockito.any()); + Mockito.doReturn(null).when(snapshotDataStoreDaoImplSpy).findByStoreSnapshot(Mockito.any(), Mockito.anyLong(), Mockito.anyLong()); for (DataStoreRole value : DataStoreRole.values()) { - Assert.assertTrue(snapshotDataStoreDaoImplSpy.expungeReferenceBySnapshotIdAndDataStoreRole(1, value)); + Assert.assertTrue(snapshotDataStoreDaoImplSpy.expungeReferenceBySnapshotIdAndDataStoreRole(1, 1, value)); } } @Test public void expungeReferenceBySnapshotIdAndDataStoreRoleTestSnapshotDataStoreIsNotNullAndExpungeIsTrueReturnTrue() { - Mockito.doReturn(searchCriteriaMock).when(snapshotDataStoreDaoImplSpy).createSearchCriteriaBySnapshotIdAndStoreRole(Mockito.anyLong(), Mockito.any()); - Mockito.doReturn(snapshotDataStoreVoMock).when(snapshotDataStoreDaoImplSpy).findOneBy(Mockito.any()); + Mockito.doReturn(snapshotDataStoreVoMock).when(snapshotDataStoreDaoImplSpy).findByStoreSnapshot(Mockito.any(), Mockito.anyLong(), Mockito.anyLong()); Mockito.doReturn(true).when(snapshotDataStoreDaoImplSpy).expunge(Mockito.anyLong()); for (DataStoreRole value : DataStoreRole.values()) { - Assert.assertTrue(snapshotDataStoreDaoImplSpy.expungeReferenceBySnapshotIdAndDataStoreRole(1, value)); + Assert.assertTrue(snapshotDataStoreDaoImplSpy.expungeReferenceBySnapshotIdAndDataStoreRole(1, 1, value)); } } @Test public void expungeReferenceBySnapshotIdAndDataStoreRoleTestSnapshotDataStoreIsNotNullAndExpungeIsFalseReturnTrue() { - Mockito.doReturn(searchCriteriaMock).when(snapshotDataStoreDaoImplSpy).createSearchCriteriaBySnapshotIdAndStoreRole(Mockito.anyLong(), Mockito.any()); - Mockito.doReturn(snapshotDataStoreVoMock).when(snapshotDataStoreDaoImplSpy).findOneBy(Mockito.any()); + Mockito.doReturn(snapshotDataStoreVoMock).when(snapshotDataStoreDaoImplSpy).findByStoreSnapshot(Mockito.any(), Mockito.anyLong(), Mockito.anyLong()); Mockito.doReturn(false).when(snapshotDataStoreDaoImplSpy).expunge(Mockito.anyLong()); for (DataStoreRole value : DataStoreRole.values()) { - Assert.assertFalse(snapshotDataStoreDaoImplSpy.expungeReferenceBySnapshotIdAndDataStoreRole(1, value)); + Assert.assertFalse(snapshotDataStoreDaoImplSpy.expungeReferenceBySnapshotIdAndDataStoreRole(1, 1, value)); } } diff --git a/engine/storage/src/test/java/org/apache/cloudstack/storage/image/db/TemplateDataStoreDaoImplTest.java b/engine/storage/src/test/java/org/apache/cloudstack/storage/image/db/TemplateDataStoreDaoImplTest.java index 0949dd97dc35..6027cfa421cd 100644 --- a/engine/storage/src/test/java/org/apache/cloudstack/storage/image/db/TemplateDataStoreDaoImplTest.java +++ b/engine/storage/src/test/java/org/apache/cloudstack/storage/image/db/TemplateDataStoreDaoImplTest.java @@ -16,14 +16,15 @@ // under the License. package org.apache.cloudstack.storage.image.db; -import com.cloud.storage.VMTemplateStorageResourceAssoc; +import java.util.Arrays; +import java.util.List; + import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; import org.junit.Assert; import org.junit.Test; -import java.util.Arrays; -import java.util.List; +import com.cloud.storage.VMTemplateStorageResourceAssoc; public class TemplateDataStoreDaoImplTest { diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/manager/PrimaryDataStoreProviderManagerImpl.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/manager/PrimaryDataStoreProviderManagerImpl.java index b799c8be389d..59ac995052f7 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/manager/PrimaryDataStoreProviderManagerImpl.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/manager/PrimaryDataStoreProviderManagerImpl.java @@ -85,4 +85,13 @@ public PrimaryDataStore getPrimaryDataStore(String uuid) { public boolean registerHostListener(String providerName, HypervisorHostListener listener) { return storageMgr.registerHostListener(providerName, listener); } + + @Override + public long getPrimaryDataStoreZoneId(long dataStoreId) { + StoragePoolVO dataStoreVO = dataStoreDao.findByIdIncludingRemoved(dataStoreId); + if (dataStoreVO == null) { + throw new CloudRuntimeException("Unable to locate datastore with id " + dataStoreId); + } + return dataStoreVO.getDataCenterId(); + } } diff --git a/engine/storage/volume/src/test/java/org/apache/cloudstack/storage/datastore/manager/PrimaryDataStoreProviderManagerImplTest.java b/engine/storage/volume/src/test/java/org/apache/cloudstack/storage/datastore/manager/PrimaryDataStoreProviderManagerImplTest.java new file mode 100644 index 000000000000..d1118ed6a89c --- /dev/null +++ b/engine/storage/volume/src/test/java/org/apache/cloudstack/storage/datastore/manager/PrimaryDataStoreProviderManagerImplTest.java @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.datastore.manager; + + +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +public class PrimaryDataStoreProviderManagerImplTest { + + @Mock + PrimaryDataStoreDao primaryDataStoreDao; + + @InjectMocks + PrimaryDataStoreProviderManagerImpl primaryDataStoreProviderManager = new PrimaryDataStoreProviderManagerImpl(); + @Test + public void testGetImageStoreZoneId() { + final long storeId = 1L; + final long zoneId = 1L; + StoragePoolVO storagePoolVO = Mockito.mock(StoragePoolVO.class); + Mockito.when(storagePoolVO.getDataCenterId()).thenReturn(zoneId); + Mockito.when(primaryDataStoreDao.findByIdIncludingRemoved(storeId)).thenReturn(storagePoolVO); + long value = primaryDataStoreProviderManager.getPrimaryDataStoreZoneId(storeId); + Assert.assertEquals(zoneId, value); + } +} diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java index 15fe75b5e233..9100ee5e34b8 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java @@ -17,6 +17,8 @@ package org.apache.cloudstack.framework.jobs.impl; +import static com.cloud.utils.HumanReadableJson.getHumanReadableBytesJson; + import java.io.Serializable; import java.util.Arrays; import java.util.Collections; @@ -33,10 +35,7 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; -import com.cloud.storage.dao.VolumeDetailsDao; import org.apache.cloudstack.api.ApiCommandResourceType; -import org.apache.log4j.Logger; -import org.apache.log4j.NDC; import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory; @@ -49,27 +48,29 @@ import org.apache.cloudstack.framework.jobs.AsyncJobExecutionContext; import org.apache.cloudstack.framework.jobs.AsyncJobManager; import org.apache.cloudstack.framework.jobs.dao.AsyncJobDao; -import org.apache.cloudstack.framework.jobs.dao.VmWorkJobDao; import org.apache.cloudstack.framework.jobs.dao.AsyncJobJoinMapDao; import org.apache.cloudstack.framework.jobs.dao.AsyncJobJournalDao; import org.apache.cloudstack.framework.jobs.dao.SyncQueueItemDao; +import org.apache.cloudstack.framework.jobs.dao.VmWorkJobDao; import org.apache.cloudstack.framework.messagebus.MessageBus; import org.apache.cloudstack.framework.messagebus.MessageDetector; import org.apache.cloudstack.framework.messagebus.PublishScope; import org.apache.cloudstack.jobs.JobInfo; import org.apache.cloudstack.jobs.JobInfo.Status; import org.apache.cloudstack.managed.context.ManagedContextRunnable; +import org.apache.cloudstack.management.ManagementServerHost; import org.apache.cloudstack.utils.identity.ManagementServerNode; +import org.apache.log4j.Logger; import org.apache.log4j.MDC; +import org.apache.log4j.NDC; import com.cloud.cluster.ClusterManagerListener; -import org.apache.cloudstack.management.ManagementServerHost; - -import com.cloud.storage.DataStoreRole; import com.cloud.storage.Snapshot; import com.cloud.storage.dao.SnapshotDao; import com.cloud.storage.dao.SnapshotDetailsDao; import com.cloud.storage.dao.SnapshotDetailsVO; +import com.cloud.storage.dao.VolumeDao; +import com.cloud.storage.dao.VolumeDetailsDao; import com.cloud.utils.DateUtil; import com.cloud.utils.Pair; import com.cloud.utils.Predicate; @@ -94,9 +95,6 @@ import com.cloud.utils.exception.ExceptionUtil; import com.cloud.utils.mgmt.JmxUtil; import com.cloud.vm.dao.VMInstanceDao; -import com.cloud.storage.dao.VolumeDao; - -import static com.cloud.utils.HumanReadableJson.getHumanReadableBytesJson; public class AsyncJobManagerImpl extends ManagerBase implements AsyncJobManager, ClusterManagerListener, Configurable { // Advanced @@ -1115,7 +1113,7 @@ public void doInTransactionWithoutResult(TransactionStatus status) { } final List snapshotList = _snapshotDetailsDao.findDetails(AsyncJob.Constants.MS_ID, Long.toString(msid), false); for (final SnapshotDetailsVO snapshotDetailsVO : snapshotList) { - SnapshotInfo snapshot = snapshotFactory.getSnapshot(snapshotDetailsVO.getResourceId(), DataStoreRole.Primary); + SnapshotInfo snapshot = snapshotFactory.getSnapshotOnPrimaryStore(snapshotDetailsVO.getResourceId()); if (snapshot == null) { _snapshotDetailsDao.remove(snapshotDetailsVO.getId()); continue; diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelper.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelper.java index 083a6fabecae..1cf4f864ab9f 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelper.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelper.java @@ -556,22 +556,33 @@ protected void loadPresetVariableValueForSnapshot(UsageVO usageRecord, Value val value.setName(snapshotVo.getName()); value.setSize(ByteScaleUtils.bytesToMebibytes(snapshotVo.getSize())); value.setSnapshotType(Snapshot.Type.values()[snapshotVo.getSnapshotType()]); - value.setStorage(getPresetVariableValueStorage(getSnapshotDataStoreId(snapshotId), usageType)); + value.setStorage(getPresetVariableValueStorage(getSnapshotDataStoreId(snapshotId, usageRecord.getZoneId()), usageType)); value.setTags(getPresetVariableValueResourceTags(snapshotId, ResourceObjectType.Snapshot)); } + protected SnapshotDataStoreVO getSnapshotImageStoreRef(long snapshotId, long zoneId) { + List snaps = snapshotDataStoreDao.listReadyBySnapshot(snapshotId, DataStoreRole.Image); + for (SnapshotDataStoreVO ref : snaps) { + ImageStoreVO store = imageStoreDao.findById(ref.getDataStoreId()); + if (store != null && zoneId == store.getDataCenterId()) { + return ref; + } + } + return null; + } + /** * If {@link SnapshotInfo#BackupSnapshotAfterTakingSnapshot} is enabled, returns the secondary storage's ID where the snapshot is. Otherwise, returns the primary storage's ID * where the snapshot is. */ - protected long getSnapshotDataStoreId(Long snapshotId) { + protected long getSnapshotDataStoreId(Long snapshotId, long zoneId) { if (backupSnapshotAfterTakingSnapshot) { - SnapshotDataStoreVO snapshotStore = snapshotDataStoreDao.findBySnapshot(snapshotId, DataStoreRole.Image); + SnapshotDataStoreVO snapshotStore = getSnapshotImageStoreRef(snapshotId, zoneId); validateIfObjectIsNull(snapshotStore, snapshotId, "data store for snapshot"); return snapshotStore.getDataStoreId(); } - SnapshotDataStoreVO snapshotStore = snapshotDataStoreDao.findBySnapshot(snapshotId, DataStoreRole.Primary); + SnapshotDataStoreVO snapshotStore = snapshotDataStoreDao.findOneBySnapshotAndDatastoreRole(snapshotId, DataStoreRole.Primary); validateIfObjectIsNull(snapshotStore, snapshotId, "data store for snapshot"); return snapshotStore.getDataStoreId(); } diff --git a/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelperTest.java b/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelperTest.java index bfc4bd463f7a..cf1a680f2bb7 100644 --- a/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelperTest.java +++ b/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelperTest.java @@ -863,7 +863,7 @@ public void loadPresetVariableValueForSnapshotTestRecordIsSnapshotSetFields() { Mockito.doReturn(expected.getName()).when(snapshotVoMock).getName(); Mockito.doReturn(expected.getSize()).when(snapshotVoMock).getSize(); Mockito.doReturn((short) 3).when(snapshotVoMock).getSnapshotType(); - Mockito.doReturn(1l).when(presetVariableHelperSpy).getSnapshotDataStoreId(Mockito.anyLong()); + Mockito.doReturn(1l).when(presetVariableHelperSpy).getSnapshotDataStoreId(Mockito.anyLong(), Mockito.anyLong()); Mockito.doReturn(expected.getStorage()).when(presetVariableHelperSpy).getPresetVariableValueStorage(Mockito.anyLong(), Mockito.anyInt()); Mockito.doReturn(expected.getTags()).when(presetVariableHelperSpy).getPresetVariableValueResourceTags(Mockito.anyLong(), Mockito.any(ResourceObjectType.class)); @@ -891,19 +891,19 @@ public void getSnapshotDataStoreIdTestDoNotBackupSnapshotToSecondaryRetrievePrim SnapshotDataStoreVO snapshotDataStoreVoMock = Mockito.mock(SnapshotDataStoreVO.class); Long expected = 1l; - Mockito.doReturn(snapshotDataStoreVoMock).when(snapshotDataStoreDaoMock).findBySnapshot(Mockito.anyLong(), Mockito.any(DataStoreRole.class)); + Mockito.doReturn(snapshotDataStoreVoMock).when(snapshotDataStoreDaoMock).findOneBySnapshotAndDatastoreRole(Mockito.anyLong(), Mockito.any(DataStoreRole.class)); Mockito.doReturn(expected).when(snapshotDataStoreVoMock).getDataStoreId(); presetVariableHelperSpy.backupSnapshotAfterTakingSnapshot = false; - Long result = presetVariableHelperSpy.getSnapshotDataStoreId(1l); + Long result = presetVariableHelperSpy.getSnapshotDataStoreId(1l, 1l); Assert.assertEquals(expected, result); Arrays.asList(DataStoreRole.values()).forEach(role -> { if (role == DataStoreRole.Primary) { - Mockito.verify(snapshotDataStoreDaoMock).findBySnapshot(Mockito.anyLong(), Mockito.eq(role)); + Mockito.verify(snapshotDataStoreDaoMock).findOneBySnapshotAndDatastoreRole(Mockito.anyLong(), Mockito.eq(role)); } else { - Mockito.verify(snapshotDataStoreDaoMock, Mockito.never()).findBySnapshot(Mockito.anyLong(), Mockito.eq(role)); + Mockito.verify(snapshotDataStoreDaoMock, Mockito.never()).findOneBySnapshotAndDatastoreRole(Mockito.anyLong(), Mockito.eq(role)); } }); } @@ -913,19 +913,22 @@ public void getSnapshotDataStoreIdTestBackupSnapshotToSecondaryRetrieveSecondary SnapshotDataStoreVO snapshotDataStoreVoMock = Mockito.mock(SnapshotDataStoreVO.class); Long expected = 2l; - Mockito.doReturn(snapshotDataStoreVoMock).when(snapshotDataStoreDaoMock).findBySnapshot(Mockito.anyLong(), Mockito.any(DataStoreRole.class)); + ImageStoreVO imageStore = Mockito.mock(ImageStoreVO.class); + Mockito.when(imageStoreDaoMock.findById(Mockito.anyLong())).thenReturn(imageStore); + Mockito.when(imageStore.getDataCenterId()).thenReturn(1L); + Mockito.when(snapshotDataStoreDaoMock.listReadyBySnapshot(Mockito.anyLong(), Mockito.any(DataStoreRole.class))).thenReturn(List.of(snapshotDataStoreVoMock)); Mockito.doReturn(expected).when(snapshotDataStoreVoMock).getDataStoreId(); presetVariableHelperSpy.backupSnapshotAfterTakingSnapshot = true; - Long result = presetVariableHelperSpy.getSnapshotDataStoreId(2l); + Long result = presetVariableHelperSpy.getSnapshotDataStoreId(2l, 1L); Assert.assertEquals(expected, result); Arrays.asList(DataStoreRole.values()).forEach(role -> { if (role == DataStoreRole.Image) { - Mockito.verify(snapshotDataStoreDaoMock).findBySnapshot(Mockito.anyLong(), Mockito.eq(role)); + Mockito.verify(snapshotDataStoreDaoMock).listReadyBySnapshot(Mockito.anyLong(), Mockito.eq(role)); } else { - Mockito.verify(snapshotDataStoreDaoMock, Mockito.never()).findBySnapshot(Mockito.anyLong(), Mockito.eq(role)); + Mockito.verify(snapshotDataStoreDaoMock, Mockito.never()).listReadyBySnapshot(Mockito.anyLong(), Mockito.eq(role)); } }); } @@ -1148,4 +1151,26 @@ public void getPresetVariableValueBackupOfferingTestSetValuesAndReturnObject() { Assert.assertEquals(expected.getExternalId(), result.getExternalId()); validateFieldNamesToIncludeInToString(Arrays.asList("id", "name", "externalId"), result); } + + @Test + public void testGetSnapshotImageStoreRefNull() { + SnapshotDataStoreVO ref1 = Mockito.mock(SnapshotDataStoreVO.class); + Mockito.when(ref1.getDataStoreId()).thenReturn(1L); + Mockito.when(snapshotDataStoreDaoMock.listReadyBySnapshot(Mockito.anyLong(), Mockito.any(DataStoreRole.class))).thenReturn(List.of(ref1)); + ImageStoreVO store = Mockito.mock(ImageStoreVO.class); + Mockito.when(store.getDataCenterId()).thenReturn(2L); + Mockito.when(imageStoreDaoMock.findById(1L)).thenReturn(store); + Assert.assertNull(presetVariableHelperSpy.getSnapshotImageStoreRef(1L, 1L)); + } + + @Test + public void testGetSnapshotImageStoreRefNotNull() { + SnapshotDataStoreVO ref1 = Mockito.mock(SnapshotDataStoreVO.class); + Mockito.when(ref1.getDataStoreId()).thenReturn(1L); + Mockito.when(snapshotDataStoreDaoMock.listReadyBySnapshot(Mockito.anyLong(), Mockito.any(DataStoreRole.class))).thenReturn(List.of(ref1)); + ImageStoreVO store = Mockito.mock(ImageStoreVO.class); + Mockito.when(store.getDataCenterId()).thenReturn(1L); + Mockito.when(imageStoreDaoMock.findById(1L)).thenReturn(store); + Assert.assertNotNull(presetVariableHelperSpy.getSnapshotImageStoreRef(1L, 1L)); + } } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageSubsystemCommandHandler.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageSubsystemCommandHandler.java index 15caa1d878ea..e56f41ea8212 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageSubsystemCommandHandler.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageSubsystemCommandHandler.java @@ -19,17 +19,25 @@ package com.cloud.storage.resource; import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; import java.util.EnumMap; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; -import com.cloud.hypervisor.vmware.manager.VmwareManager; -import com.cloud.utils.NumbersUtil; -import org.apache.log4j.Logger; import org.apache.cloudstack.storage.command.CopyCmdAnswer; import org.apache.cloudstack.storage.command.CopyCommand; import org.apache.cloudstack.storage.command.DeleteCommand; +import org.apache.cloudstack.storage.command.QuerySnapshotZoneCopyAnswer; +import org.apache.cloudstack.storage.command.QuerySnapshotZoneCopyCommand; import org.apache.cloudstack.storage.to.SnapshotObjectTO; import org.apache.cloudstack.storage.to.TemplateObjectTO; import org.apache.cloudstack.storage.to.VolumeObjectTO; +import org.apache.log4j.Logger; import com.cloud.agent.api.Answer; import com.cloud.agent.api.to.DataObjectType; @@ -38,9 +46,11 @@ import com.cloud.agent.api.to.NfsTO; import com.cloud.agent.api.to.S3TO; import com.cloud.agent.api.to.SwiftTO; +import com.cloud.hypervisor.vmware.manager.VmwareManager; import com.cloud.hypervisor.vmware.manager.VmwareStorageManager; import com.cloud.storage.DataStoreRole; import com.cloud.storage.resource.VmwareStorageProcessor.VmwareStorageProcessorConfigurableFields; +import com.cloud.utils.NumbersUtil; public class VmwareStorageSubsystemCommandHandler extends StorageSubsystemCommandHandlerBase { @@ -202,4 +212,32 @@ protected Answer execute(CopyCommand cmd) { } } + @Override + protected Answer execute(QuerySnapshotZoneCopyCommand cmd) { + SnapshotObjectTO snapshot = cmd.getSnapshot(); + String parentPath = storageResource.getRootDir(snapshot.getDataStore().getUrl(), _nfsVersion); + String path = snapshot.getPath(); + File snapFile = new File(parentPath + File.separator + path); + if (snapFile.exists() && !snapFile.isDirectory()) { + return new QuerySnapshotZoneCopyAnswer(cmd, List.of(path)); + } + int index = path.lastIndexOf(File.separator); + String snapDir = path.substring(0, index); + List files = new ArrayList<>(); + try (Stream stream = Files.list(Paths.get(parentPath + File.separator + snapDir))) { + List fileNames = stream + .filter(file -> !Files.isDirectory(file)) + .map(Path::getFileName) + .map(Path::toString) + .collect(Collectors.toList()); + for (String file : fileNames) { + file = snapDir + "/" + file; + s_logger.debug(String.format("Found snapshot file %s", file)); + files.add(file); + } + } catch (IOException ioe) { + s_logger.error("Error preparing file list for snapshot copy", ioe); + } + return new QuerySnapshotZoneCopyAnswer(cmd, files); + } } diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java index cad88dcdd15e..d37a339eb2d9 100644 --- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java +++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java @@ -22,13 +22,6 @@ import javax.inject.Inject; -import com.cloud.agent.api.storage.MigrateVolumeCommand; -import com.cloud.agent.api.storage.ResizeVolumeCommand; -import com.cloud.agent.api.to.StorageFilerTO; -import com.cloud.host.HostVO; -import com.cloud.vm.VMInstanceVO; -import com.cloud.vm.VirtualMachine; -import com.cloud.vm.dao.VMInstanceDao; import org.apache.cloudstack.engine.subsystem.api.storage.ChapInfo; import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult; import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult; @@ -69,13 +62,17 @@ import org.apache.log4j.Logger; import com.cloud.agent.api.Answer; +import com.cloud.agent.api.storage.MigrateVolumeCommand; +import com.cloud.agent.api.storage.ResizeVolumeCommand; import com.cloud.agent.api.to.DataObjectType; import com.cloud.agent.api.to.DataStoreTO; import com.cloud.agent.api.to.DataTO; import com.cloud.agent.api.to.DiskTO; +import com.cloud.agent.api.to.StorageFilerTO; import com.cloud.alert.AlertManager; import com.cloud.configuration.Config; import com.cloud.host.Host; +import com.cloud.host.HostVO; import com.cloud.host.dao.HostDao; import com.cloud.server.ManagementServerImpl; import com.cloud.storage.DataStoreRole; @@ -97,7 +94,10 @@ import com.cloud.utils.NumbersUtil; import com.cloud.utils.Pair; import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.vm.VMInstanceVO; +import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachineManager; +import com.cloud.vm.dao.VMInstanceDao; import com.google.common.base.Preconditions; public class ScaleIOPrimaryDataStoreDriver implements PrimaryDataStoreDriver { @@ -915,7 +915,7 @@ public void updateSnapshotsAfterCopyVolume(DataObject srcData, DataObject destDa List snapshots = snapshotDao.listByVolumeId(srcVolumeId); if (CollectionUtils.isNotEmpty(snapshots)) { for (SnapshotVO snapshot : snapshots) { - SnapshotDataStoreVO snapshotStore = snapshotDataStoreDao.findBySnapshot(snapshot.getId(), DataStoreRole.Primary); + SnapshotDataStoreVO snapshotStore = snapshotDataStoreDao.findByStoreSnapshot(DataStoreRole.Primary, srcPoolId, snapshot.getId()); if (snapshotStore == null) { continue; } @@ -1086,7 +1086,7 @@ private Answer migrateVolume(DataObject srcData, DataObject destData) { List snapshots = snapshotDao.listByVolumeId(srcData.getId()); if (CollectionUtils.isNotEmpty(snapshots)) { for (SnapshotVO snapshot : snapshots) { - SnapshotDataStoreVO snapshotStore = snapshotDataStoreDao.findBySnapshot(snapshot.getId(), DataStoreRole.Primary); + SnapshotDataStoreVO snapshotStore = snapshotDataStoreDao.findByStoreSnapshot(DataStoreRole.Primary, srcPoolId, snapshot.getId()); if (snapshotStore == null) { continue; } diff --git a/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/driver/StorPoolPrimaryDataStoreDriver.java b/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/driver/StorPoolPrimaryDataStoreDriver.java index 22ad73a118a3..0b8777c3b757 100644 --- a/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/driver/StorPoolPrimaryDataStoreDriver.java +++ b/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/driver/StorPoolPrimaryDataStoreDriver.java @@ -18,6 +18,52 @@ */ package org.apache.cloudstack.storage.datastore.driver; +import java.util.List; +import java.util.Map; + +import javax.inject.Inject; + +import org.apache.cloudstack.engine.subsystem.api.storage.ChapInfo; +import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult; +import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult; +import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; +import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; +import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector; +import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreDriver; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; +import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; +import org.apache.cloudstack.framework.async.AsyncCompletionCallback; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.storage.RemoteHostEndPoint; +import org.apache.cloudstack.storage.command.CommandResult; +import org.apache.cloudstack.storage.command.CopyCmdAnswer; +import org.apache.cloudstack.storage.command.CreateObjectAnswer; +import org.apache.cloudstack.storage.command.StorageSubSystemCommand; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; +import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailVO; +import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; +import org.apache.cloudstack.storage.datastore.util.StorPoolHelper; +import org.apache.cloudstack.storage.datastore.util.StorPoolUtil; +import org.apache.cloudstack.storage.datastore.util.StorPoolUtil.SpApiResponse; +import org.apache.cloudstack.storage.datastore.util.StorPoolUtil.SpConnectionDesc; +import org.apache.cloudstack.storage.snapshot.StorPoolConfigurationManager; +import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; +import org.apache.cloudstack.storage.to.SnapshotObjectTO; +import org.apache.cloudstack.storage.to.TemplateObjectTO; +import org.apache.cloudstack.storage.to.VolumeObjectTO; +import org.apache.cloudstack.storage.volume.VolumeObject; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.collections4.MapUtils; +import org.apache.log4j.Logger; + import com.cloud.agent.api.Answer; import com.cloud.agent.api.storage.ResizeVolumeAnswer; import com.cloud.agent.api.storage.StorPoolBackupSnapshotCommand; @@ -61,50 +107,6 @@ import com.cloud.vm.VirtualMachine.State; import com.cloud.vm.VirtualMachineManager; import com.cloud.vm.dao.VMInstanceDao; -import org.apache.cloudstack.engine.subsystem.api.storage.ChapInfo; -import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult; -import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult; -import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; -import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; -import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; -import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector; -import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreDriver; -import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; -import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo; -import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; -import org.apache.cloudstack.framework.async.AsyncCompletionCallback; -import org.apache.cloudstack.framework.config.dao.ConfigurationDao; -import org.apache.cloudstack.storage.RemoteHostEndPoint; -import org.apache.cloudstack.storage.command.CommandResult; -import org.apache.cloudstack.storage.command.CopyCmdAnswer; -import org.apache.cloudstack.storage.command.CreateObjectAnswer; -import org.apache.cloudstack.storage.command.StorageSubSystemCommand; -import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; -import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; -import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; -import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailVO; -import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao; -import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; -import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; -import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; -import org.apache.cloudstack.storage.datastore.util.StorPoolHelper; -import org.apache.cloudstack.storage.datastore.util.StorPoolUtil; -import org.apache.cloudstack.storage.datastore.util.StorPoolUtil.SpApiResponse; -import org.apache.cloudstack.storage.datastore.util.StorPoolUtil.SpConnectionDesc; -import org.apache.cloudstack.storage.snapshot.StorPoolConfigurationManager; -import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; -import org.apache.cloudstack.storage.to.SnapshotObjectTO; -import org.apache.cloudstack.storage.to.TemplateObjectTO; -import org.apache.cloudstack.storage.to.VolumeObjectTO; -import org.apache.cloudstack.storage.volume.VolumeObject; -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.collections4.MapUtils; -import org.apache.log4j.Logger; - -import javax.inject.Inject; - -import java.util.List; -import java.util.Map; public class StorPoolPrimaryDataStoreDriver implements PrimaryDataStoreDriver { @@ -142,6 +144,18 @@ public class StorPoolPrimaryDataStoreDriver implements PrimaryDataStoreDriver { private StoragePoolDetailsDao storagePoolDetailsDao; @Inject private StoragePoolHostDao storagePoolHostDao; + @Inject + DataStoreManager dataStoreManager; + + private SnapshotDataStoreVO getSnapshotImageStoreRef(long snapshotId, long zoneId) { + List snaps = snapshotDataStoreDao.listReadyBySnapshot(snapshotId, DataStoreRole.Image); + for (SnapshotDataStoreVO ref : snaps) { + if (zoneId == dataStoreManager.getStoreZoneId(ref.getDataStoreId(), ref.getRole())) { + return ref; + } + } + return null; + } @Override public Map getCapabilities() { @@ -468,7 +482,7 @@ public void copyAsync(DataObject srcData, DataObject dstData, AsyncCompletionCal } else if (resp.getError().getName().equals("objectDoesNotExist")) { //check if snapshot is on secondary storage StorPoolUtil.spLog("Snapshot %s does not exists on StorPool, will try to create a volume from a snopshot on secondary storage", snapshotName); - SnapshotDataStoreVO snap = snapshotDataStoreDao.findBySnapshot(sinfo.getId(), DataStoreRole.Image); + SnapshotDataStoreVO snap = getSnapshotImageStoreRef(sinfo.getId(), vinfo.getDataCenterId()); if (snap != null && StorPoolStorageAdaptor.getVolumeNameFromPath(snap.getInstallPath(), false) == null) { resp = StorPoolUtil.volumeCreate(srcData.getUuid(), null, size, null, "no", "snapshot", sinfo.getBaseVolume().getMaxIops(), conn); if (resp.getError() == null) { diff --git a/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/snapshot/StorPoolSnapshotStrategy.java b/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/snapshot/StorPoolSnapshotStrategy.java index 4808ee24e139..55d691f33e0b 100644 --- a/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/snapshot/StorPoolSnapshotStrategy.java +++ b/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/snapshot/StorPoolSnapshotStrategy.java @@ -16,10 +16,13 @@ // under the License. package org.apache.cloudstack.storage.snapshot; +import java.util.ArrayList; import java.util.List; import javax.inject.Inject; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine.Event; import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine.State; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory; @@ -36,6 +39,7 @@ import org.apache.cloudstack.storage.datastore.util.StorPoolUtil; import org.apache.cloudstack.storage.datastore.util.StorPoolUtil.SpApiResponse; import org.apache.cloudstack.storage.datastore.util.StorPoolUtil.SpConnectionDesc; +import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -48,6 +52,7 @@ import com.cloud.storage.dao.SnapshotDao; import com.cloud.storage.dao.SnapshotDetailsDao; import com.cloud.storage.dao.SnapshotDetailsVO; +import com.cloud.storage.dao.SnapshotZoneDao; import com.cloud.storage.dao.VolumeDao; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.fsm.NoTransitionException; @@ -73,6 +78,10 @@ public class StorPoolSnapshotStrategy implements SnapshotStrategy { private SnapshotDataFactory snapshotDataFactory; @Inject private StoragePoolDetailsDao storagePoolDetailsDao; + @Inject + DataStoreManager dataStoreMgr; + @Inject + SnapshotZoneDao snapshotZoneDao; @Override public SnapshotInfo backupSnapshot(SnapshotInfo snapshotInfo) { @@ -92,7 +101,7 @@ public SnapshotInfo backupSnapshot(SnapshotInfo snapshotInfo) { } @Override - public boolean deleteSnapshot(Long snapshotId) { + public boolean deleteSnapshot(Long snapshotId, Long zoneId) { final SnapshotVO snapshotVO = _snapshotDao.findById(snapshotId); VolumeVO volume = _volumeDao.findByIdIncludingRemoved(snapshotVO.getVolumeId()); @@ -108,11 +117,7 @@ public boolean deleteSnapshot(Long snapshotId) { final String err = String.format("Failed to clean-up Storpool snapshot %s. Error: %s", name, resp.getError()); StorPoolUtil.spLog(err); } else { - SnapshotDetailsVO snapshotDetails = _snapshotDetailsDao.findDetail(snapshotId, snapshotVO.getUuid()); - if (snapshotDetails != null) { - _snapshotDetailsDao.removeDetails(snapshotId); - } - res = deleteSnapshotFromDb(snapshotId); + res = deleteSnapshotFromDbIfNeeded(snapshotVO, zoneId); StorPoolUtil.spLog("StorpoolSnapshotStrategy.deleteSnapshot: executed successfully=%s, snapshot uuid=%s, name=%s", res, snapshotVO.getUuid(), name); } } catch (Exception e) { @@ -125,13 +130,22 @@ public boolean deleteSnapshot(Long snapshotId) { } @Override - public StrategyPriority canHandle(Snapshot snapshot, SnapshotOperation op) { + public StrategyPriority canHandle(Snapshot snapshot, Long zoneId, SnapshotOperation op) { log.debug(String.format("StorpoolSnapshotStrategy.canHandle: snapshot=%s, uuid=%s, op=%s", snapshot.getName(), snapshot.getUuid(), op)); if (op != SnapshotOperation.DELETE) { return StrategyPriority.CANT_HANDLE; } - + SnapshotDataStoreVO snapshotOnPrimary = _snapshotStoreDao.findOneBySnapshotAndDatastoreRole(snapshot.getId(), DataStoreRole.Primary); + if (snapshotOnPrimary == null) { + return StrategyPriority.CANT_HANDLE; + } + if (zoneId != null) { // If zoneId is present, then it should be same as the zoneId of primary store + StoragePoolVO storagePoolVO = _primaryDataStoreDao.findById(snapshotOnPrimary.getDataStoreId()); + if (!zoneId.equals(storagePoolVO.getDataCenterId())) { + return StrategyPriority.CANT_HANDLE; + } + } String name = StorPoolHelper.getSnapshotName(snapshot.getId(), snapshot.getUuid(), _snapshotStoreDao, _snapshotDetailsDao); if (name != null) { StorPoolUtil.spLog("StorpoolSnapshotStrategy.canHandle: globalId=%s", name); @@ -147,6 +161,7 @@ public StrategyPriority canHandle(Snapshot snapshot, SnapshotOperation op) { private boolean deleteSnapshotChain(SnapshotInfo snapshot) { log.debug("delete snapshot chain for snapshot: " + snapshot.getId()); + final SnapshotInfo snapOnImage = snapshot; boolean result = false; boolean resultIsSet = false; try { @@ -174,8 +189,7 @@ private boolean deleteSnapshotChain(SnapshotInfo snapshot) { } } if (!deleted) { - SnapshotInfo snap = snapshotDataFactory.getSnapshot(snapshot.getId(), DataStoreRole.Image); - if (StorPoolStorageAdaptor.getVolumeNameFromPath(snap.getPath(), true) == null) { + if (StorPoolStorageAdaptor.getVolumeNameFromPath(snapOnImage.getPath(), true) == null) { try { boolean r = snapshotSvr.deleteSnapshot(snapshot); if (r) { @@ -204,8 +218,64 @@ private boolean deleteSnapshotChain(SnapshotInfo snapshot) { return result; } - private boolean deleteSnapshotFromDb(Long snapshotId) { - SnapshotVO snapshotVO = _snapshotDao.findById(snapshotId); + protected boolean areLastSnapshotRef(long snapshotId) { + List snapshotStoreRefs = _snapshotStoreDao.findBySnapshotId(snapshotId); + if (CollectionUtils.isEmpty(snapshotStoreRefs) || snapshotStoreRefs.size() == 1) { + return true; + } + return snapshotStoreRefs.size() == 2 && DataStoreRole.Primary.equals(snapshotStoreRefs.get(1).getRole()); + } + + protected boolean deleteSnapshotOnImageAndPrimary(long snapshotId, DataStore store) { + SnapshotInfo snapshotOnImage = snapshotDataFactory.getSnapshot(snapshotId, store); + SnapshotObject obj = (SnapshotObject)snapshotOnImage; + boolean areLastSnapshotRef = areLastSnapshotRef(snapshotId); + try { + if (areLastSnapshotRef) { + obj.processEvent(Snapshot.Event.DestroyRequested); + } + } catch (NoTransitionException e) { + log.debug("Failed to set the state to destroying: ", e); + return false; + } + + try { + boolean result = deleteSnapshotChain(snapshotOnImage); + _snapshotStoreDao.updateDisplayForSnapshotStoreRole(snapshotId, store.getId(), store.getRole(), false); + if (areLastSnapshotRef) { + obj.processEvent(Snapshot.Event.OperationSucceeded); + } + if (result) { + SnapshotDataStoreVO snapshotOnPrimary = _snapshotStoreDao.findOneBySnapshotAndDatastoreRole(snapshotOnImage.getSnapshotId(), DataStoreRole.Primary); + if (snapshotOnPrimary != null) { + snapshotOnPrimary.setState(State.Destroyed); + _snapshotStoreDao.update(snapshotOnPrimary.getId(), snapshotOnPrimary); + } + } + } catch (Exception e) { + log.debug("Failed to delete snapshot: ", e); + try { + if (areLastSnapshotRef) { + obj.processEvent(Snapshot.Event.OperationFailed); + } + } catch (NoTransitionException e1) { + log.debug("Failed to change snapshot state: " + e.toString()); + } + return false; + } + return true; + } + + private boolean deleteSnapshotFromDbIfNeeded(SnapshotVO snapshotVO, Long zoneId) { + final long snapshotId = snapshotVO.getId(); + SnapshotDetailsVO snapshotDetails = _snapshotDetailsDao.findDetail(snapshotId, snapshotVO.getUuid()); + if (snapshotDetails != null) { + _snapshotDetailsDao.removeDetails(snapshotId); + } + + if (zoneId != null && List.of(Snapshot.State.Allocated, Snapshot.State.CreatedOnPrimary).contains(snapshotVO.getState())) { + throw new InvalidParameterValueException(String.format("Snapshot in %s can not be deleted for a zone", snapshotVO.getState())); + } if (snapshotVO.getState() == Snapshot.State.Allocated) { _snapshotDao.remove(snapshotId); @@ -218,10 +288,21 @@ private boolean deleteSnapshotFromDb(Long snapshotId) { if (Snapshot.State.Error.equals(snapshotVO.getState())) { List storeRefs = _snapshotStoreDao.findBySnapshotId(snapshotId); + List deletedRefs = new ArrayList<>(); for (SnapshotDataStoreVO ref : storeRefs) { - _snapshotStoreDao.expunge(ref.getId()); + boolean refZoneIdMatch = false; + if (zoneId != null) { + Long refZoneId = dataStoreMgr.getStoreZoneId(ref.getDataStoreId(), ref.getRole()); + refZoneIdMatch = zoneId.equals(refZoneId); + } + if (zoneId == null || refZoneIdMatch) { + _snapshotStoreDao.expunge(ref.getId()); + deletedRefs.add(ref.getId()); + } + } + if (deletedRefs.size() == storeRefs.size()) { + _snapshotDao.remove(snapshotId); } - _snapshotDao.remove(snapshotId); return true; } @@ -233,46 +314,26 @@ private boolean deleteSnapshotFromDb(Long snapshotId) { if (!Snapshot.State.BackedUp.equals(snapshotVO.getState()) && !Snapshot.State.Error.equals(snapshotVO.getState()) && !Snapshot.State.Destroying.equals(snapshotVO.getState())) { - throw new InvalidParameterValueException("Can't delete snapshotshot " + snapshotId + " due to it is in " + snapshotVO.getState() + " Status"); - } - - SnapshotInfo snapshotOnImage = snapshotDataFactory.getSnapshot(snapshotId, DataStoreRole.Image); - if (snapshotOnImage == null) { - log.debug("Can't find snapshot on backup storage, delete it in db"); - _snapshotDao.remove(snapshotId); - return true; + throw new InvalidParameterValueException("Can't delete snapshot " + snapshotId + " due to it is in " + snapshotVO.getState() + " Status"); } - - SnapshotObject obj = (SnapshotObject)snapshotOnImage; - try { - obj.processEvent(Snapshot.Event.DestroyRequested); - } catch (NoTransitionException e) { - log.debug("Failed to set the state to destroying: ", e); - return false; + List storeRefs = _snapshotStoreDao.listReadyBySnapshot(snapshotId, DataStoreRole.Image); + if (zoneId != null) { + storeRefs.removeIf(ref -> !zoneId.equals(dataStoreMgr.getStoreZoneId(ref.getDataStoreId(), ref.getRole()))); } - - try { - boolean result = deleteSnapshotChain(snapshotOnImage); - obj.processEvent(Snapshot.Event.OperationSucceeded); - if (result) { - SnapshotDataStoreVO snapshotOnPrimary = _snapshotStoreDao.findBySnapshot(snapshotId, DataStoreRole.Primary); - if (snapshotOnPrimary != null) { - snapshotOnPrimary.setState(State.Destroyed); - _snapshotStoreDao.update(snapshotOnPrimary.getId(), snapshotOnPrimary); - } + for (SnapshotDataStoreVO ref : storeRefs) { + if (!deleteSnapshotOnImageAndPrimary(snapshotId, dataStoreMgr.getDataStore(ref.getDataStoreId(), ref.getRole()))) { + return false; } - } catch (Exception e) { - log.debug("Failed to delete snapshot: ", e); - try { - obj.processEvent(Snapshot.Event.OperationFailed); - } catch (NoTransitionException e1) { - log.debug("Failed to change snapshot state: " + e.toString()); - } - return false; + } + if (zoneId != null) { + snapshotZoneDao.removeSnapshotFromZone(snapshotVO.getId(), zoneId); + } else { + snapshotZoneDao.removeSnapshotFromZones(snapshotVO.getId()); } return true; } + @Override public SnapshotInfo takeSnapshot(SnapshotInfo snapshot) { return null; diff --git a/scripts/storage/secondary/createvolume.sh b/scripts/storage/secondary/createvolume.sh index 91370dff710d..e5838aea5f09 100755 --- a/scripts/storage/secondary/createvolume.sh +++ b/scripts/storage/secondary/createvolume.sh @@ -18,8 +18,8 @@ -# $Id: createtmplt.sh 9132 2010-06-04 20:17:43Z manuel $ $HeadURL: svn://svn.lab.vmops.com/repos/vmdev/java/scripts/storage/secondary/createtmplt.sh $ -# createtmplt.sh -- install a volume +# $Id: createvolume.sh 9132 2010-06-04 20:17:43Z manuel $ $HeadURL: svn://svn.lab.vmops.com/repos/vmdev/java/scripts/storage/secondary/createvolume.sh $ +# createvolume.sh -- install a volume usage() { printf "Usage: %s: -t -n -f -c -d -h [-u] [-v]\n" $(basename $0) >&2 diff --git a/server/src/main/java/com/cloud/api/ApiDBUtils.java b/server/src/main/java/com/cloud/api/ApiDBUtils.java index 38c7b99150d3..d30e8b829208 100644 --- a/server/src/main/java/com/cloud/api/ApiDBUtils.java +++ b/server/src/main/java/com/cloud/api/ApiDBUtils.java @@ -16,6 +16,78 @@ // under the License. package com.cloud.api; +import java.util.ArrayList; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.List; +import java.util.ListIterator; +import java.util.Map; +import java.util.Set; + +import javax.annotation.PostConstruct; +import javax.inject.Inject; + +import org.apache.cloudstack.acl.Role; +import org.apache.cloudstack.acl.RoleService; +import org.apache.cloudstack.affinity.AffinityGroup; +import org.apache.cloudstack.affinity.AffinityGroupResponse; +import org.apache.cloudstack.affinity.dao.AffinityGroupDao; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiConstants.DomainDetails; +import org.apache.cloudstack.api.ApiConstants.HostDetails; +import org.apache.cloudstack.api.ApiConstants.VMDetails; +import org.apache.cloudstack.api.ResponseObject.ResponseView; +import org.apache.cloudstack.api.response.AccountResponse; +import org.apache.cloudstack.api.response.AsyncJobResponse; +import org.apache.cloudstack.api.response.BackupOfferingResponse; +import org.apache.cloudstack.api.response.BackupResponse; +import org.apache.cloudstack.api.response.BackupScheduleResponse; +import org.apache.cloudstack.api.response.DiskOfferingResponse; +import org.apache.cloudstack.api.response.DomainResponse; +import org.apache.cloudstack.api.response.DomainRouterResponse; +import org.apache.cloudstack.api.response.EventResponse; +import org.apache.cloudstack.api.response.HostForMigrationResponse; +import org.apache.cloudstack.api.response.HostResponse; +import org.apache.cloudstack.api.response.HostTagResponse; +import org.apache.cloudstack.api.response.ImageStoreResponse; +import org.apache.cloudstack.api.response.InstanceGroupResponse; +import org.apache.cloudstack.api.response.NetworkOfferingResponse; +import org.apache.cloudstack.api.response.ProjectAccountResponse; +import org.apache.cloudstack.api.response.ProjectInvitationResponse; +import org.apache.cloudstack.api.response.ProjectResponse; +import org.apache.cloudstack.api.response.ResourceIconResponse; +import org.apache.cloudstack.api.response.ResourceTagResponse; +import org.apache.cloudstack.api.response.SecurityGroupResponse; +import org.apache.cloudstack.api.response.ServiceOfferingResponse; +import org.apache.cloudstack.api.response.SnapshotResponse; +import org.apache.cloudstack.api.response.StoragePoolResponse; +import org.apache.cloudstack.api.response.StorageTagResponse; +import org.apache.cloudstack.api.response.TemplateResponse; +import org.apache.cloudstack.api.response.UserResponse; +import org.apache.cloudstack.api.response.UserVmResponse; +import org.apache.cloudstack.api.response.VolumeResponse; +import org.apache.cloudstack.api.response.VpcOfferingResponse; +import org.apache.cloudstack.api.response.ZoneResponse; +import org.apache.cloudstack.backup.Backup; +import org.apache.cloudstack.backup.BackupOffering; +import org.apache.cloudstack.backup.BackupSchedule; +import org.apache.cloudstack.backup.dao.BackupDao; +import org.apache.cloudstack.backup.dao.BackupOfferingDao; +import org.apache.cloudstack.backup.dao.BackupScheduleDao; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; +import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.framework.jobs.AsyncJob; +import org.apache.cloudstack.framework.jobs.AsyncJobManager; +import org.apache.cloudstack.framework.jobs.dao.AsyncJobDao; +import org.apache.cloudstack.resourcedetail.SnapshotPolicyDetailVO; +import org.apache.cloudstack.resourcedetail.dao.DiskOfferingDetailsDao; +import org.apache.cloudstack.resourcedetail.dao.SnapshotPolicyDetailsDao; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; + import com.cloud.agent.api.VgpuTypesInfo; import com.cloud.api.query.dao.AccountJoinDao; import com.cloud.api.query.dao.AffinityGroupJoinDao; @@ -35,6 +107,7 @@ import com.cloud.api.query.dao.ResourceTagJoinDao; import com.cloud.api.query.dao.SecurityGroupJoinDao; import com.cloud.api.query.dao.ServiceOfferingJoinDao; +import com.cloud.api.query.dao.SnapshotJoinDao; import com.cloud.api.query.dao.StoragePoolJoinDao; import com.cloud.api.query.dao.TemplateJoinDao; import com.cloud.api.query.dao.UserAccountJoinDao; @@ -60,6 +133,7 @@ import com.cloud.api.query.vo.ResourceTagJoinVO; import com.cloud.api.query.vo.SecurityGroupJoinVO; import com.cloud.api.query.vo.ServiceOfferingJoinVO; +import com.cloud.api.query.vo.SnapshotJoinVO; import com.cloud.api.query.vo.StoragePoolJoinVO; import com.cloud.api.query.vo.TemplateJoinVO; import com.cloud.api.query.vo.UserAccountJoinVO; @@ -274,72 +348,6 @@ import com.cloud.vm.dao.VMInstanceDao; import com.cloud.vm.snapshot.VMSnapshot; import com.cloud.vm.snapshot.dao.VMSnapshotDao; -import org.apache.cloudstack.acl.Role; -import org.apache.cloudstack.acl.RoleService; -import org.apache.cloudstack.affinity.AffinityGroup; -import org.apache.cloudstack.affinity.AffinityGroupResponse; -import org.apache.cloudstack.affinity.dao.AffinityGroupDao; -import org.apache.cloudstack.api.ApiCommandResourceType; -import org.apache.cloudstack.api.ApiConstants.DomainDetails; -import org.apache.cloudstack.api.ApiConstants.HostDetails; -import org.apache.cloudstack.api.ApiConstants.VMDetails; -import org.apache.cloudstack.api.ResponseObject.ResponseView; -import org.apache.cloudstack.api.response.AccountResponse; -import org.apache.cloudstack.api.response.AsyncJobResponse; -import org.apache.cloudstack.api.response.BackupOfferingResponse; -import org.apache.cloudstack.api.response.BackupResponse; -import org.apache.cloudstack.api.response.BackupScheduleResponse; -import org.apache.cloudstack.api.response.DiskOfferingResponse; -import org.apache.cloudstack.api.response.DomainResponse; -import org.apache.cloudstack.api.response.DomainRouterResponse; -import org.apache.cloudstack.api.response.EventResponse; -import org.apache.cloudstack.api.response.HostForMigrationResponse; -import org.apache.cloudstack.api.response.HostResponse; -import org.apache.cloudstack.api.response.HostTagResponse; -import org.apache.cloudstack.api.response.ImageStoreResponse; -import org.apache.cloudstack.api.response.InstanceGroupResponse; -import org.apache.cloudstack.api.response.NetworkOfferingResponse; -import org.apache.cloudstack.api.response.ProjectAccountResponse; -import org.apache.cloudstack.api.response.ProjectInvitationResponse; -import org.apache.cloudstack.api.response.ProjectResponse; -import org.apache.cloudstack.api.response.ResourceIconResponse; -import org.apache.cloudstack.api.response.ResourceTagResponse; -import org.apache.cloudstack.api.response.SecurityGroupResponse; -import org.apache.cloudstack.api.response.ServiceOfferingResponse; -import org.apache.cloudstack.api.response.StoragePoolResponse; -import org.apache.cloudstack.api.response.StorageTagResponse; -import org.apache.cloudstack.api.response.TemplateResponse; -import org.apache.cloudstack.api.response.UserResponse; -import org.apache.cloudstack.api.response.UserVmResponse; -import org.apache.cloudstack.api.response.VolumeResponse; -import org.apache.cloudstack.api.response.VpcOfferingResponse; -import org.apache.cloudstack.api.response.ZoneResponse; -import org.apache.cloudstack.backup.Backup; -import org.apache.cloudstack.backup.BackupOffering; -import org.apache.cloudstack.backup.BackupSchedule; -import org.apache.cloudstack.backup.dao.BackupDao; -import org.apache.cloudstack.backup.dao.BackupOfferingDao; -import org.apache.cloudstack.backup.dao.BackupScheduleDao; -import org.apache.cloudstack.context.CallContext; -import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; -import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService; -import org.apache.cloudstack.framework.config.dao.ConfigurationDao; -import org.apache.cloudstack.framework.jobs.AsyncJob; -import org.apache.cloudstack.framework.jobs.AsyncJobManager; -import org.apache.cloudstack.framework.jobs.dao.AsyncJobDao; -import org.apache.cloudstack.resourcedetail.dao.DiskOfferingDetailsDao; -import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; -import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; - -import javax.annotation.PostConstruct; -import javax.inject.Inject; -import java.util.ArrayList; -import java.util.EnumSet; -import java.util.HashMap; -import java.util.List; -import java.util.ListIterator; -import java.util.Map; -import java.util.Set; public class ApiDBUtils { private static ManagementServer s_ms; @@ -441,6 +449,7 @@ public class ApiDBUtils { static AccountJoinDao s_accountJoinDao; static AsyncJobJoinDao s_jobJoinDao; static TemplateJoinDao s_templateJoinDao; + static SnapshotJoinDao s_snapshotJoinDao; static PhysicalNetworkTrafficTypeDao s_physicalNetworkTrafficTypeDao; static PhysicalNetworkServiceProviderDao s_physicalNetworkServiceProviderDao; @@ -471,6 +480,7 @@ public class ApiDBUtils { static BackupOfferingDao s_backupOfferingDao; static NicDao s_nicDao; static ResourceManagerUtil s_resourceManagerUtil; + static SnapshotPolicyDetailsDao s_snapshotPolicyDetailsDao; @Inject private ManagementServer ms; @@ -662,6 +672,8 @@ public class ApiDBUtils { private AsyncJobJoinDao jobJoinDao; @Inject private TemplateJoinDao templateJoinDao; + @Inject + private SnapshotJoinDao snapshotJoinDao; @Inject private PhysicalNetworkTrafficTypeDao physicalNetworkTrafficTypeDao; @@ -725,6 +737,8 @@ public class ApiDBUtils { private ResourceIconDao resourceIconDao; @Inject private ResourceManagerUtil resourceManagerUtil; + @Inject + SnapshotPolicyDetailsDao snapshotPolicyDetailsDao; @PostConstruct void init() { @@ -820,6 +834,7 @@ void init() { s_accountJoinDao = accountJoinDao; s_jobJoinDao = jobJoinDao; s_templateJoinDao = templateJoinDao; + s_snapshotJoinDao = snapshotJoinDao; s_physicalNetworkTrafficTypeDao = physicalNetworkTrafficTypeDao; s_physicalNetworkServiceProviderDao = physicalNetworkServiceProviderDao; @@ -832,6 +847,7 @@ void init() { s_vpcOfferingDao = vpcOfferingDao; s_vpcOfferingJoinDao = vpcOfferingJoinDao; s_snapshotPolicyDao = snapshotPolicyDao; + s_snapshotPolicyDetailsDao = snapshotPolicyDetailsDao; s_asyncJobDao = asyncJobDao; s_hostDetailsDao = hostDetailsDao; s_clusterDetailsDao = clusterDetailsDao; @@ -1649,6 +1665,20 @@ public static SnapshotPolicy findSnapshotPolicyById(long policyId) { return s_snapshotPolicyDao.findById(policyId); } + public static List findSnapshotPolicyZones(SnapshotPolicy policy, Volume volume) { + List zoneDetails = s_snapshotPolicyDetailsDao.findDetails(policy.getId(), ApiConstants.ZONE_ID); + List zoneIds = new ArrayList<>(); + for (SnapshotPolicyDetailVO detail : zoneDetails) { + try { + zoneIds.add(Long.valueOf(detail.getValue())); + } catch (NumberFormatException ignored) {} + } + if (volume != null && !zoneIds.contains(volume.getDataCenterId())) { + zoneIds.add(0, volume.getDataCenterId()); + } + return s_zoneDao.listByIds(zoneIds); + } + public static VpcOffering findVpcOfferingById(long offeringId) { return s_vpcOfferingDao.findById(offeringId); } @@ -2083,6 +2113,10 @@ public static TemplateResponse newTemplateResponse(EnumSet detail return s_templateJoinDao.newTemplateResponse(detailsView, view, vr); } + public static SnapshotResponse newSnapshotResponse(ResponseView view, boolean isShowUnique, SnapshotJoinVO vr) { + return s_snapshotJoinDao.newSnapshotResponse(view, isShowUnique, vr); + } + public static TemplateResponse newIsoResponse(TemplateJoinVO vr) { return s_templateJoinDao.newIsoResponse(vr); } @@ -2091,6 +2125,10 @@ public static TemplateResponse fillTemplateDetails(EnumSet detail return s_templateJoinDao.setTemplateResponse(detailsView, view, vrData, vr); } + public static SnapshotResponse fillSnapshotDetails(SnapshotResponse vrData, SnapshotJoinVO vr) { + return s_snapshotJoinDao.setSnapshotResponse(vrData, vr); + } + public static List newTemplateView(VirtualMachineTemplate vr) { return s_templateJoinDao.newTemplateView(vr); } diff --git a/server/src/main/java/com/cloud/api/ApiResponseHelper.java b/server/src/main/java/com/cloud/api/ApiResponseHelper.java index 5b723a51cc88..7d80cd1a6daf 100644 --- a/server/src/main/java/com/cloud/api/ApiResponseHelper.java +++ b/server/src/main/java/com/cloud/api/ApiResponseHelper.java @@ -38,7 +38,6 @@ import javax.inject.Inject; -import com.cloud.hypervisor.Hypervisor; import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.acl.ControlledEntity.ACLType; import org.apache.cloudstack.affinity.AffinityGroup; @@ -263,6 +262,7 @@ import com.cloud.host.ControlState; import com.cloud.host.Host; import com.cloud.host.HostVO; +import com.cloud.hypervisor.Hypervisor; import com.cloud.hypervisor.HypervisorCapabilities; import com.cloud.network.GuestVlan; import com.cloud.network.GuestVlanRange; @@ -646,6 +646,7 @@ public SnapshotResponse createSnapshotResponse(Snapshot snapshot) { DataCenter zone = ApiDBUtils.findZoneById(volume.getDataCenterId()); if (zone != null) { snapshotResponse.setZoneId(zone.getUuid()); + snapshotResponse.setZoneName(zone.getName()); } if (volume.getVolumeType() == Volume.Type.ROOT && volume.getInstanceId() != null) { @@ -673,7 +674,7 @@ public SnapshotResponse createSnapshotResponse(Snapshot snapshot) { } else { DataStoreRole dataStoreRole = getDataStoreRole(snapshot, _snapshotStoreDao, _dataStoreMgr); - snapshotInfo = snapshotfactory.getSnapshot(snapshot.getId(), dataStoreRole); + snapshotInfo = snapshotfactory.getSnapshotWithRoleAndZone(snapshot.getId(), dataStoreRole, volume.getDataCenterId()); } if (snapshotInfo == null) { @@ -681,7 +682,7 @@ public SnapshotResponse createSnapshotResponse(Snapshot snapshot) { snapshotResponse.setRevertable(false); } else { snapshotResponse.setRevertable(snapshotInfo.isRevertable()); - snapshotResponse.setPhysicaSize(snapshotInfo.getPhysicalSize()); + snapshotResponse.setPhysicalSize(snapshotInfo.getPhysicalSize()); } // set tag information @@ -700,7 +701,7 @@ public SnapshotResponse createSnapshotResponse(Snapshot snapshot) { } public static DataStoreRole getDataStoreRole(Snapshot snapshot, SnapshotDataStoreDao snapshotStoreDao, DataStoreManager dataStoreMgr) { - SnapshotDataStoreVO snapshotStore = snapshotStoreDao.findBySnapshot(snapshot.getId(), DataStoreRole.Primary); + SnapshotDataStoreVO snapshotStore = snapshotStoreDao.findOneBySnapshotAndDatastoreRole(snapshot.getId(), DataStoreRole.Primary); if (snapshotStore == null) { return DataStoreRole.Image; @@ -807,6 +808,16 @@ public SnapshotPolicyResponse createSnapshotPolicyResponse(SnapshotPolicy policy CollectionUtils.addIgnoreNull(tagResponses, tagResponse); } policyResponse.setTags(new HashSet<>(tagResponses)); + List zoneResponses = new ArrayList<>(); + List zones = ApiDBUtils.findSnapshotPolicyZones(policy, vol); + for (DataCenterVO zone : zones) { + ZoneResponse zoneResponse = new ZoneResponse(); + zoneResponse.setId(zone.getUuid()); + zoneResponse.setName(zone.getName()); + zoneResponse.setTags(null); + zoneResponses.add(zoneResponse); + } + policyResponse.setZones(new HashSet<>(zoneResponses)); return policyResponse; } @@ -1936,7 +1947,7 @@ public List createTemplateResponses(ResponseView view, long te // it seems that the volume can actually be removed from the DB at some point if it's deleted // if volume comes back null, use another technique to try to discover the zone if (volume == null) { - SnapshotDataStoreVO snapshotStore = _snapshotStoreDao.findBySnapshot(snapshot.getId(), DataStoreRole.Primary); + SnapshotDataStoreVO snapshotStore = _snapshotStoreDao.findOneBySnapshotAndDatastoreRole(snapshot.getId(), DataStoreRole.Primary); if (snapshotStore != null) { long storagePoolId = snapshotStore.getDataStoreId(); @@ -2837,6 +2848,23 @@ private void populateOwner(ControlledEntityResponse response, ControlledEntity o response.setDomainName(domain.getName()); } + private void populateOwner(ControlledViewEntityResponse response, ControlledEntity object) { + Account account = ApiDBUtils.findAccountById(object.getAccountId()); + + if (account.getType() == Account.Type.PROJECT) { + // find the project + Project project = ApiDBUtils.findProjectByProjectAccountId(account.getId()); + response.setProjectId(project.getUuid()); + response.setProjectName(project.getName()); + } else { + response.setAccountName(account.getAccountName()); + } + + Domain domain = ApiDBUtils.findDomainById(object.getDomainId()); + response.setDomainId(domain.getUuid()); + response.setDomainName(domain.getName()); + } + public static void populateOwner(ControlledViewEntityResponse response, ControlledViewEntity object) { if (object.getAccountType() == Account.Type.PROJECT) { diff --git a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java index a50bf0738670..491104b654c9 100644 --- a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java +++ b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java @@ -79,6 +79,8 @@ import org.apache.cloudstack.api.command.user.project.ListProjectsCmd; import org.apache.cloudstack.api.command.user.resource.ListDetailOptionsCmd; import org.apache.cloudstack.api.command.user.securitygroup.ListSecurityGroupsCmd; +import org.apache.cloudstack.api.command.user.snapshot.CopySnapshotCmd; +import org.apache.cloudstack.api.command.user.snapshot.ListSnapshotsCmd; import org.apache.cloudstack.api.command.user.tag.ListTagsCmd; import org.apache.cloudstack.api.command.user.template.ListTemplatesCmd; import org.apache.cloudstack.api.command.user.vm.ListVMsCmd; @@ -108,6 +110,7 @@ import org.apache.cloudstack.api.response.RouterHealthCheckResultResponse; import org.apache.cloudstack.api.response.SecurityGroupResponse; import org.apache.cloudstack.api.response.ServiceOfferingResponse; +import org.apache.cloudstack.api.response.SnapshotResponse; import org.apache.cloudstack.api.response.StoragePoolResponse; import org.apache.cloudstack.api.response.StorageTagResponse; import org.apache.cloudstack.api.response.TemplateResponse; @@ -154,6 +157,7 @@ import com.cloud.api.query.dao.ResourceTagJoinDao; import com.cloud.api.query.dao.SecurityGroupJoinDao; import com.cloud.api.query.dao.ServiceOfferingJoinDao; +import com.cloud.api.query.dao.SnapshotJoinDao; import com.cloud.api.query.dao.StoragePoolJoinDao; import com.cloud.api.query.dao.TemplateJoinDao; import com.cloud.api.query.dao.UserAccountJoinDao; @@ -178,6 +182,7 @@ import com.cloud.api.query.vo.ResourceTagJoinVO; import com.cloud.api.query.vo.SecurityGroupJoinVO; import com.cloud.api.query.vo.ServiceOfferingJoinVO; +import com.cloud.api.query.vo.SnapshotJoinVO; import com.cloud.api.query.vo.StoragePoolJoinVO; import com.cloud.api.query.vo.TemplateJoinVO; import com.cloud.api.query.vo.UserAccountJoinVO; @@ -228,6 +233,8 @@ import com.cloud.storage.DataStoreRole; import com.cloud.storage.DiskOfferingVO; import com.cloud.storage.ScopeType; +import com.cloud.storage.Snapshot; +import com.cloud.storage.SnapshotVO; import com.cloud.storage.Storage; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.Storage.TemplateType; @@ -236,6 +243,7 @@ import com.cloud.storage.VMTemplateVO; import com.cloud.storage.Volume; import com.cloud.storage.VolumeApiServiceImpl; +import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.DiskOfferingDao; import com.cloud.storage.dao.StoragePoolTagsDao; import com.cloud.storage.dao.VMTemplateDao; @@ -461,6 +469,9 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q @Inject private ManagementServerHostDao msHostDao; + @Inject + private SnapshotJoinDao snapshotJoinDao; + @Inject EntityManager entityManager; @@ -3392,6 +3403,7 @@ private Pair, Integer> listDataCentersInternal(ListZonesC Account account = CallContext.current().getCallingAccount(); Long domainId = cmd.getDomainId(); Long id = cmd.getId(); + List ids = getIdsListFromCmd(cmd.getId(), cmd.getIds()); String keyword = cmd.getKeyword(); String name = cmd.getName(); String networkType = cmd.getNetworkType(); @@ -3418,6 +3430,10 @@ private Pair, Integer> listDataCentersInternal(ListZonesC sc.addAnd("networkType", SearchCriteria.Op.EQ, networkType); } + if (CollectionUtils.isNotEmpty(ids)) { + sc.addAnd("id", SearchCriteria.Op.IN, ids.toArray()); + } + if (id != null) { sc.addAnd("id", SearchCriteria.Op.EQ, id); } else if (name != null) { @@ -4496,6 +4512,190 @@ public List listRouterHealthChecks(GetRouterHea return responseGenerator.createHealthCheckResponse(_routerDao.findById(routerId), result); } + @Override + public ListResponse listSnapshots(ListSnapshotsCmd cmd) { + Account caller = CallContext.current().getCallingAccount(); + Pair, Integer> result = searchForSnapshotsWithParams(cmd.getId(), cmd.getIds(), + cmd.getVolumeId(), cmd.getSnapshotName(), cmd.getKeyword(), cmd.getTags(), + cmd.getSnapshotType(), cmd.getIntervalType(), cmd.getZoneId(), cmd.getLocationType(), + cmd.isShowUnique(), cmd.getAccountName(), cmd.getDomainId(), cmd.getProjectId(), + cmd.getStartIndex(), cmd.getPageSizeVal(), cmd.listAll(), cmd.isRecursive(), caller); + ListResponse response = new ListResponse<>(); + ResponseView respView = ResponseView.Restricted; + if (CallContext.current().getCallingAccount().getType() == Account.Type.ADMIN) { + respView = ResponseView.Full; + } + List templateResponses = ViewResponseHelper.createSnapshotResponse(respView, cmd.isShowUnique(), result.first().toArray(new SnapshotJoinVO[result.first().size()])); + response.setResponses(templateResponses, result.second()); + return response; + } + + @Override + public SnapshotResponse listSnapshot(CopySnapshotCmd cmd) { + Account caller = CallContext.current().getCallingAccount(); + List zoneIds = cmd.getDestinationZoneIds(); + Pair, Integer> result = searchForSnapshotsWithParams(cmd.getId(), null, + null, null, null, null, + null, null, zoneIds.get(0), Snapshot.LocationType.SECONDARY.name(), + false, null, null, null, + null, null, true, false, caller); + ResponseView respView = ResponseView.Restricted; + if (CallContext.current().getCallingAccount().getType() == Account.Type.ADMIN) { + respView = ResponseView.Full; + } + List templateResponses = ViewResponseHelper.createSnapshotResponse(respView, false, result.first().get(0)); + return templateResponses.get(0); + } + + + + private Pair, Integer> searchForSnapshotsWithParams(final Long id, List ids, + final Long volumeId, final String name, final String keyword, final Map tags, + final String snapshotTypeStr, final String intervalTypeStr, final Long zoneId, final String locationTypeStr, + final boolean isShowUnique, final String accountName, Long domainId, final Long projectId, + final Long startIndex, final Long pageSize,final boolean listAll, boolean isRecursive, final Account caller) { + ids = getIdsListFromCmd(id, ids); + Snapshot.LocationType locationType = null; + if (locationTypeStr != null) { + try { + locationType = Snapshot.LocationType.valueOf(locationTypeStr.trim().toUpperCase()); + } catch (IllegalArgumentException e) { + throw new InvalidParameterValueException(String.format("Invalid %s specified, %s", ApiConstants.LOCATION_TYPE, locationTypeStr)); + } + } + + Filter searchFilter = new Filter(SnapshotJoinVO.class, "snapshotStorePair", SortKeyAscending.value(), startIndex, pageSize); + + List permittedAccountIds = new ArrayList<>(); + Ternary domainIdRecursiveListProject = new Ternary(domainId, isRecursive, null); + _accountMgr.buildACLSearchParameters(caller, id, accountName, projectId, permittedAccountIds, domainIdRecursiveListProject, listAll, false); + ListProjectResourcesCriteria listProjectResourcesCriteria = domainIdRecursiveListProject.third(); + domainId = domainIdRecursiveListProject.first(); + isRecursive = domainIdRecursiveListProject.second(); + // Verify parameters + if (volumeId != null) { + VolumeVO volume = volumeDao.findById(volumeId); + if (volume != null) { + _accountMgr.checkAccess(CallContext.current().getCallingAccount(), null, true, volume); + } + } + + SearchBuilder sb = snapshotJoinDao.createSearchBuilder(); + if (isShowUnique) { + sb.select(null, Func.DISTINCT, sb.entity().getId()); // select distinct snapshotId + } else { + sb.select(null, Func.DISTINCT, sb.entity().getSnapshotStorePair()); // select distinct (snapshotId, store_role, store_id) key + } + _accountMgr.buildACLSearchBuilder(sb, domainId, isRecursive, permittedAccountIds, listProjectResourcesCriteria); + sb.and("statusNEQ", sb.entity().getStatus(), SearchCriteria.Op.NEQ); //exclude those Destroyed snapshot, not showing on UI + sb.and("volumeId", sb.entity().getVolumeId(), SearchCriteria.Op.EQ); + sb.and("name", sb.entity().getName(), SearchCriteria.Op.EQ); + sb.and("id", sb.entity().getId(), SearchCriteria.Op.EQ); + sb.and("idIN", sb.entity().getId(), SearchCriteria.Op.IN); + sb.and("snapshotTypeEQ", sb.entity().getSnapshotType(), SearchCriteria.Op.IN); + sb.and("snapshotTypeNEQ", sb.entity().getSnapshotType(), SearchCriteria.Op.NIN); + sb.and("dataCenterId", sb.entity().getDataCenterId(), SearchCriteria.Op.EQ); + sb.and("locationType", sb.entity().getStoreRole(), SearchCriteria.Op.EQ); + + if (tags != null && !tags.isEmpty()) { + SearchBuilder tagSearch = _resourceTagDao.createSearchBuilder(); + for (int count = 0; count < tags.size(); count++) { + tagSearch.or().op("key" + String.valueOf(count), tagSearch.entity().getKey(), SearchCriteria.Op.EQ); + tagSearch.and("value" + String.valueOf(count), tagSearch.entity().getValue(), SearchCriteria.Op.EQ); + tagSearch.cp(); + } + tagSearch.and("resourceType", tagSearch.entity().getResourceType(), SearchCriteria.Op.EQ); + sb.groupBy(sb.entity().getId()); + sb.join("tagSearch", tagSearch, sb.entity().getId(), tagSearch.entity().getResourceId(), JoinBuilder.JoinType.INNER); + } + + SearchCriteria sc = sb.create(); + _accountMgr.buildACLSearchCriteria(sc, domainId, isRecursive, permittedAccountIds, listProjectResourcesCriteria); + + sc.setParameters("statusNEQ", Snapshot.State.Destroyed); + + if (volumeId != null) { + sc.setParameters("volumeId", volumeId); + } + + if (tags != null && !tags.isEmpty()) { + int count = 0; + sc.setJoinParameters("tagSearch", "resourceType", ResourceObjectType.Snapshot.toString()); + for (String key : tags.keySet()) { + sc.setJoinParameters("tagSearch", "key" + String.valueOf(count), key); + sc.setJoinParameters("tagSearch", "value" + String.valueOf(count), tags.get(key)); + count++; + } + } + + if (zoneId != null) { + sc.setParameters("dataCenterId", zoneId); + } + + setIdsListToSearchCriteria(sc, ids); + + if (name != null) { + sc.setParameters("name", name); + } + + if (id != null) { + sc.setParameters("id", id); + } + + if (locationType != null) { + sc.setParameters("locationType", Snapshot.LocationType.PRIMARY.equals(locationType) ? locationType.name() : DataStoreRole.Image.name()); + } + + if (keyword != null) { + SearchCriteria ssc = snapshotJoinDao.createSearchCriteria(); + ssc.addOr("name", SearchCriteria.Op.LIKE, "%" + keyword + "%"); + sc.addAnd("name", SearchCriteria.Op.SC, ssc); + } + + if (snapshotTypeStr != null) { + Snapshot.Type snapshotType = SnapshotVO.getSnapshotType(snapshotTypeStr); + if (snapshotType == null) { + throw new InvalidParameterValueException("Unsupported snapshot type " + snapshotTypeStr); + } + if (snapshotType == Snapshot.Type.RECURRING) { + sc.setParameters("snapshotTypeEQ", Snapshot.Type.HOURLY.ordinal(), Snapshot.Type.DAILY.ordinal(), Snapshot.Type.WEEKLY.ordinal(), Snapshot.Type.MONTHLY.ordinal()); + } else { + sc.setParameters("snapshotTypeEQ", snapshotType.ordinal()); + } + } else if (intervalTypeStr != null && volumeId != null) { + Snapshot.Type type = SnapshotVO.getSnapshotType(intervalTypeStr); + if (type == null) { + throw new InvalidParameterValueException("Unsupported snapshot interval type " + intervalTypeStr); + } + sc.setParameters("snapshotTypeEQ", type.ordinal()); + } else { + // Show only MANUAL and RECURRING snapshot types + sc.setParameters("snapshotTypeNEQ", Snapshot.Type.TEMPLATE.ordinal(), Snapshot.Type.GROUP.ordinal()); + } + + Pair, Integer> snapshotDataPair; + if (isShowUnique) { + snapshotDataPair = snapshotJoinDao.searchAndDistinctCount(sc, searchFilter, new String[]{"snapshot_view.id"}); + } else { + snapshotDataPair = snapshotJoinDao.searchAndDistinctCount(sc, searchFilter, new String[]{"snapshot_view.snapshot_store_pair"}); + } + + Integer count = snapshotDataPair.second(); + if (count == 0) { + // empty result + return snapshotDataPair; + } + List snapshotData = snapshotDataPair.first(); + List snapshots; + if (isShowUnique) { + snapshots = snapshotJoinDao.findByDistinctIds(zoneId, snapshotData.stream().map(SnapshotJoinVO::getId).toArray(Long[]::new)); + } else { + snapshots = snapshotJoinDao.searchBySnapshotStorePair(snapshotData.stream().map(SnapshotJoinVO::getSnapshotStorePair).toArray(String[]::new)); + } + + return new Pair<>(snapshots, count); + } + @Override public String getConfigComponentName() { return QueryService.class.getSimpleName(); diff --git a/server/src/main/java/com/cloud/api/query/ViewResponseHelper.java b/server/src/main/java/com/cloud/api/query/ViewResponseHelper.java index 48031425bb8e..b415699ab19d 100644 --- a/server/src/main/java/com/cloud/api/query/ViewResponseHelper.java +++ b/server/src/main/java/com/cloud/api/query/ViewResponseHelper.java @@ -49,6 +49,7 @@ import org.apache.cloudstack.api.response.ResourceTagResponse; import org.apache.cloudstack.api.response.SecurityGroupResponse; import org.apache.cloudstack.api.response.ServiceOfferingResponse; +import org.apache.cloudstack.api.response.SnapshotResponse; import org.apache.cloudstack.api.response.StoragePoolResponse; import org.apache.cloudstack.api.response.StorageTagResponse; import org.apache.cloudstack.api.response.TemplateResponse; @@ -78,6 +79,7 @@ import com.cloud.api.query.vo.ResourceTagJoinVO; import com.cloud.api.query.vo.SecurityGroupJoinVO; import com.cloud.api.query.vo.ServiceOfferingJoinVO; +import com.cloud.api.query.vo.SnapshotJoinVO; import com.cloud.api.query.vo.StoragePoolJoinVO; import com.cloud.api.query.vo.TemplateJoinVO; import com.cloud.api.query.vo.UserAccountJoinVO; @@ -592,6 +594,23 @@ public static List createTemplateResponse(EnumSet(vrDataList.values()); } + public static List createSnapshotResponse(ResponseView view, boolean isShowUnique, SnapshotJoinVO... snapshots) { + LinkedHashMap vrDataList = new LinkedHashMap<>(); + for (SnapshotJoinVO vr : snapshots) { + SnapshotResponse vrData = vrDataList.get(vr.getSnapshotStorePair()); + if (vrData == null) { + // first time encountering this snapshot + vrData = ApiDBUtils.newSnapshotResponse(view, isShowUnique, vr); + } + else{ + // update tags + vrData = ApiDBUtils.fillSnapshotDetails(vrData, vr); + } + vrDataList.put(vr.getSnapshotStorePair(), vrData); + } + return new ArrayList(vrDataList.values()); + } + public static List createTemplateUpdateResponse(ResponseView view, TemplateJoinVO... templates) { LinkedHashMap vrDataList = new LinkedHashMap<>(); for (TemplateJoinVO vr : templates) { diff --git a/server/src/main/java/com/cloud/api/query/dao/SnapshotJoinDao.java b/server/src/main/java/com/cloud/api/query/dao/SnapshotJoinDao.java new file mode 100644 index 000000000000..4e916e66ae78 --- /dev/null +++ b/server/src/main/java/com/cloud/api/query/dao/SnapshotJoinDao.java @@ -0,0 +1,41 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.api.query.dao; + +import java.util.List; + +import org.apache.cloudstack.api.ResponseObject; +import org.apache.cloudstack.api.response.SnapshotResponse; + +import com.cloud.api.query.vo.SnapshotJoinVO; +import com.cloud.utils.Pair; +import com.cloud.utils.db.Filter; +import com.cloud.utils.db.GenericDao; +import com.cloud.utils.db.SearchCriteria; + +public interface SnapshotJoinDao extends GenericDao { + + SnapshotResponse newSnapshotResponse(ResponseObject.ResponseView view, boolean isShowUnique, SnapshotJoinVO snapshotJoinVO); + + SnapshotResponse setSnapshotResponse(SnapshotResponse snapshotResponse, SnapshotJoinVO snapshot); + + Pair, Integer> searchIncludingRemovedAndCount(final SearchCriteria sc, final Filter filter); + + List searchBySnapshotStorePair(String... pairs); + List findByDistinctIds(Long zoneId, Long... ids); +} diff --git a/server/src/main/java/com/cloud/api/query/dao/SnapshotJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/SnapshotJoinDaoImpl.java new file mode 100644 index 000000000000..a913dd7f568a --- /dev/null +++ b/server/src/main/java/com/cloud/api/query/dao/SnapshotJoinDaoImpl.java @@ -0,0 +1,248 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.api.query.dao; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.inject.Inject; + +import org.apache.cloudstack.annotation.AnnotationService; +import org.apache.cloudstack.annotation.dao.AnnotationDao; +import org.apache.cloudstack.api.ResponseObject; +import org.apache.cloudstack.api.response.SnapshotResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.query.QueryService; +import org.apache.log4j.Logger; + +import com.cloud.api.ApiResponseHelper; +import com.cloud.api.query.vo.SnapshotJoinVO; +import com.cloud.storage.Snapshot; +import com.cloud.storage.VMTemplateStorageResourceAssoc; +import com.cloud.user.Account; +import com.cloud.user.AccountService; +import com.cloud.utils.Pair; +import com.cloud.utils.db.Filter; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; + +public class SnapshotJoinDaoImpl extends GenericDaoBaseWithTagInformation implements SnapshotJoinDao { + + public static final Logger s_logger = Logger.getLogger(SnapshotJoinDaoImpl.class); + + @Inject + private AccountService accountService; + @Inject + private AnnotationDao annotationDao; + @Inject + private ConfigurationDao configDao; + @Inject + SnapshotDataFactory snapshotDataFactory; + + private final SearchBuilder snapshotStorePairSearch; + + private final SearchBuilder snapshotIdsSearch; + + SnapshotJoinDaoImpl() { + snapshotStorePairSearch = createSearchBuilder(); + snapshotStorePairSearch.and("snapshotStoreState", snapshotStorePairSearch.entity().getStoreState(), SearchCriteria.Op.IN); + snapshotStorePairSearch.and("snapshotStoreIdIN", snapshotStorePairSearch.entity().getSnapshotStorePair(), SearchCriteria.Op.IN); + snapshotStorePairSearch.done(); + + snapshotIdsSearch = createSearchBuilder(); + snapshotIdsSearch.and("zoneId", snapshotIdsSearch.entity().getDataCenterId(), SearchCriteria.Op.EQ); + snapshotIdsSearch.and("idsIN", snapshotIdsSearch.entity().getId(), SearchCriteria.Op.IN); + snapshotIdsSearch.groupBy(snapshotIdsSearch.entity().getId()); + snapshotIdsSearch.done(); + } + + private void setSnapshotInfoDetailsInResponse(SnapshotJoinVO snapshot, SnapshotResponse snapshotResponse, boolean isShowUnique) { + if (!isShowUnique) { + return; + } + if (snapshot.getDataCenterId() == null) { + return; + } + SnapshotInfo snapshotInfo = null; + snapshotInfo = snapshotDataFactory.getSnapshotWithRoleAndZone(snapshot.getId(), snapshot.getStoreRole(), snapshot.getDataCenterId()); + if (snapshotInfo == null) { + s_logger.debug("Unable to find info for image store snapshot with uuid " + snapshot.getUuid()); + snapshotResponse.setRevertable(false); + } else { + snapshotResponse.setRevertable(snapshotInfo.isRevertable()); + snapshotResponse.setPhysicalSize(snapshotInfo.getPhysicalSize()); + } + } + + private String getSnapshotStatus(SnapshotJoinVO snapshot) { + String status = snapshot.getStatus().toString(); + if (snapshot.getDownloadState() == null) { + return status; + } + if (snapshot.getDownloadState() != VMTemplateStorageResourceAssoc.Status.DOWNLOADED) { + status = "Processing"; + if (snapshot.getDownloadState() == VMTemplateStorageResourceAssoc.Status.DOWNLOAD_IN_PROGRESS) { + status = snapshot.getDownloadPercent() + "% Downloaded"; + } else if (snapshot.getErrorString() == null) { + status = snapshot.getStoreState().toString(); + } else { + status = snapshot.getErrorString(); + } + } + return status; + } + + @Override + public SnapshotResponse newSnapshotResponse(ResponseObject.ResponseView view, boolean isShowUnique, SnapshotJoinVO snapshot) { + final Account caller = CallContext.current().getCallingAccount(); + SnapshotResponse snapshotResponse = new SnapshotResponse(); + snapshotResponse.setId(snapshot.getUuid()); + // populate owner. + ApiResponseHelper.populateOwner(snapshotResponse, snapshot); + if (snapshot.getVolumeId() != null) { + snapshotResponse.setVolumeId(snapshot.getVolumeUuid()); + snapshotResponse.setVolumeName(snapshot.getVolumeName()); + snapshotResponse.setVolumeType(snapshot.getVolumeType().name()); + snapshotResponse.setVirtualSize(snapshot.getVolumeSize()); + } + snapshotResponse.setZoneId(snapshot.getDataCenterUuid()); + snapshotResponse.setZoneName(snapshot.getDataCenterName()); + snapshotResponse.setCreated(snapshot.getCreated()); + snapshotResponse.setName(snapshot.getName()); + String intervalType = null; + if (snapshot.getSnapshotType() >= 0 && snapshot.getSnapshotType() < Snapshot.Type.values().length) { + intervalType = Snapshot.Type.values()[snapshot.getSnapshotType()].name(); + } + snapshotResponse.setIntervalType(intervalType); + snapshotResponse.setState(snapshot.getStatus()); + snapshotResponse.setLocationType(snapshot.getLocationType() != null ? snapshot.getLocationType().name() : null); + if (!isShowUnique) { + snapshotResponse.setDatastoreState(snapshot.getStoreState() != null ? snapshot.getStoreState().name() : null); + if (view.equals(ResponseObject.ResponseView.Full)) { + snapshotResponse.setDatastoreId(snapshot.getStoreUuid()); + snapshotResponse.setDatastoreName(snapshot.getStoreName()); + snapshotResponse.setDatastoreType(snapshot.getStoreRole() != null ? snapshot.getStoreRole().name() : null); + } + // If the user is an 'Admin' or 'the owner of template' or template belongs to a project, add the template download status + if (view == ResponseObject.ResponseView.Full || + snapshot.getAccountId() == caller.getId() || + snapshot.getAccountType() == Account.Type.PROJECT) { + String status = getSnapshotStatus(snapshot); + if (status != null) { + snapshotResponse.setStatus(status); + } + } + Map downloadDetails = new HashMap<>(); + downloadDetails.put("downloadPercent", Integer.toString(snapshot.getDownloadPercent())); + downloadDetails.put("downloadState", (snapshot.getDownloadState() != null ? snapshot.getDownloadState().toString() : "")); + snapshotResponse.setDownloadDetails(downloadDetails); + } + setSnapshotInfoDetailsInResponse(snapshot, snapshotResponse, isShowUnique); + setSnapshotResponse(snapshotResponse, snapshot); + + snapshotResponse.setObjectName("snapshot"); + return snapshotResponse; + } + + @Override + public SnapshotResponse setSnapshotResponse(SnapshotResponse snapshotResponse, SnapshotJoinVO snapshot) { + // update tag information + long tag_id = snapshot.getTagId(); + if (tag_id > 0) { + addTagInformation(snapshot, snapshotResponse); + } + + if (snapshotResponse.hasAnnotation() == null) { + snapshotResponse.setHasAnnotation(annotationDao.hasAnnotations(snapshot.getUuid(), AnnotationService.EntityType.SNAPSHOT.name(), + accountService.isRootAdmin(CallContext.current().getCallingAccount().getId()))); + } + return snapshotResponse; + } + + @Override + public Pair, Integer> searchIncludingRemovedAndCount(final SearchCriteria sc, final Filter filter) { + List objects = searchIncludingRemoved(sc, filter, null, false); + Integer count = getDistinctCount(sc); + return new Pair<>(objects, count); + } + + @Override + public List searchBySnapshotStorePair(String... pairs) { + // set detail batch query size + int DETAILS_BATCH_SIZE = 2000; + String batchCfg = configDao.getValue("detail.batch.query.size"); + if (batchCfg != null) { + DETAILS_BATCH_SIZE = Integer.parseInt(batchCfg); + } + // query details by batches + Filter searchFilter = new Filter(SnapshotJoinVO.class, "snapshotStorePair", QueryService.SortKeyAscending.value(), null, null); + List uvList = new ArrayList<>(); + // query details by batches + int curr_index = 0; + if (pairs.length > DETAILS_BATCH_SIZE) { + while ((curr_index + DETAILS_BATCH_SIZE) <= pairs.length) { + String[] labels = new String[DETAILS_BATCH_SIZE]; + for (int k = 0, j = curr_index; j < curr_index + DETAILS_BATCH_SIZE; j++, k++) { + labels[k] = pairs[j]; + } + SearchCriteria sc = snapshotStorePairSearch.create(); + sc.setParameters("snapshotStoreIdIN", labels); + List snaps = searchIncludingRemoved(sc, searchFilter, null, false); + if (snaps != null) { + uvList.addAll(snaps); + } + curr_index += DETAILS_BATCH_SIZE; + } + } + if (curr_index < pairs.length) { + int batch_size = (pairs.length - curr_index); + String[] labels = new String[batch_size]; + for (int k = 0, j = curr_index; j < curr_index + batch_size; j++, k++) { + labels[k] = pairs[j]; + } + SearchCriteria sc = snapshotStorePairSearch.create(); + sc.setParameters("snapshotStoreIdIN", labels); + List vms = searchIncludingRemoved(sc, searchFilter, null, false); + if (vms != null) { + uvList.addAll(vms); + } + } + return uvList; + } + + @Override + public List findByDistinctIds(Long zoneId, Long... ids) { + if (ids == null || ids.length == 0) { + return new ArrayList<>(); + } + + Filter searchFilter = new Filter(SnapshotJoinVO.class, "snapshotStorePair", QueryService.SortKeyAscending.value(), null, null); + + SearchCriteria sc = snapshotIdsSearch.create(); + if (zoneId != null) { + sc.setParameters("zoneId", zoneId); + } + sc.setParameters("idsIN", ids); + return searchIncludingRemoved(sc, searchFilter, null, false); + } +} diff --git a/server/src/main/java/com/cloud/api/query/vo/SnapshotJoinVO.java b/server/src/main/java/com/cloud/api/query/vo/SnapshotJoinVO.java new file mode 100644 index 000000000000..9ec74dac1288 --- /dev/null +++ b/server/src/main/java/com/cloud/api/query/vo/SnapshotJoinVO.java @@ -0,0 +1,352 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.api.query.vo; + +import java.util.Date; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; +import javax.persistence.Table; + +import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; + +import com.cloud.hypervisor.Hypervisor; +import com.cloud.storage.DataStoreRole; +import com.cloud.storage.Snapshot; +import com.cloud.storage.VMTemplateStorageResourceAssoc; +import com.cloud.storage.Volume; +import com.cloud.user.Account; +import com.cloud.utils.db.GenericDao; + +@Entity +@Table(name = "snapshot_view") +public class SnapshotJoinVO extends BaseViewWithTagInformationVO implements ControlledViewEntity { + @Column(name = "uuid") + private String uuid; + + @Column(name = "name") + private String name; + + @Column(name = "status") + @Enumerated(value = EnumType.STRING) + private Snapshot.State status; + + @Column(name = "disk_offering_id") + Long diskOfferingId; + + @Column(name = "snapshot_type") + short snapshotType; + + @Column(name = "type_description") + String typeDescription; + + @Column(name = "size") + long size; + + @Column(name = GenericDao.CREATED_COLUMN) + Date created; + + @Column(name = GenericDao.REMOVED_COLUMN) + Date removed; + + @Column(name = "location_type") + @Enumerated(value = EnumType.STRING) + private Snapshot.LocationType locationType; + + @Column(name = "hypervisor_type") + @Enumerated(value = EnumType.STRING) + Hypervisor.HypervisorType hypervisorType; + + @Column(name = "account_id") + private long accountId; + + @Column(name = "account_uuid") + private String accountUuid; + + @Column(name = "account_name") + private String accountName = null; + + @Column(name = "account_type") + @Enumerated(value = EnumType.ORDINAL) + private Account.Type accountType; + + @Column(name = "domain_id") + private long domainId; + + @Column(name = "domain_uuid") + private String domainUuid; + + @Column(name = "domain_name") + private String domainName = null; + + @Column(name = "domain_path") + private String domainPath = null; + + @Column(name = "project_id") + private Long projectId; + + @Column(name = "project_uuid") + private String projectUuid; + + @Column(name = "project_name") + private String projectName; + + @Column(name = "data_center_id") + private Long dataCenterId; + + @Column(name = "data_center_uuid") + private String dataCenterUuid; + + @Column(name = "data_center_name") + private String dataCenterName; + + @Column(name = "volume_id") + private Long volumeId; + + @Column(name = "volume_uuid") + private String volumeUuid; + + @Column(name = "volume_name") + private String volumeName; + + @Column(name = "volume_type") + @Enumerated(EnumType.STRING) + Volume.Type volumeType = Volume.Type.UNKNOWN; + + @Column(name = "volume_size") + Long volumeSize; + + @Column(name = "store_id") + private Long storeId; + + @Column(name = "store_uuid") + private String storeUuid; + + @Column(name = "store_name") + private String storeName; + + @Column(name = "store_role") + @Enumerated(EnumType.STRING) + private DataStoreRole storeRole; + + @Column(name = "store_state") + @Enumerated(EnumType.STRING) + private ObjectInDataStoreStateMachine.State storeState; + + @Column(name = "download_state") + @Enumerated(EnumType.STRING) + private VMTemplateStorageResourceAssoc.Status downloadState; + + @Column(name = "download_pct") + private int downloadPercent; + + @Column(name = "error_str") + private String errorString; + + @Column(name = "store_size") + private long storeSize; + + @Column(name = "created_on_store") + private Date createdOnStore = null; + + @Column(name = "snapshot_store_pair") + private String snapshotStorePair; + + @Override + public String getUuid() { + return uuid; + } + + @Override + public String getName() { + return name; + } + + public Snapshot.State getStatus() { + return status; + } + + public Long getDiskOfferingId() { + return diskOfferingId; + } + + public short getSnapshotType() { + return snapshotType; + } + + public String getTypeDescription() { + return typeDescription; + } + + public long getSize() { + return size; + } + + public Date getCreated() { + return created; + } + + public Date getRemoved() { + return removed; + } + + public Snapshot.LocationType getLocationType() { + return locationType; + } + + public Hypervisor.HypervisorType getHypervisorType() { + return hypervisorType; + } + + @Override + public long getAccountId() { + return accountId; + } + + @Override + public String getAccountUuid() { + return accountUuid; + } + + @Override + public String getAccountName() { + return accountName; + } + + @Override + public Account.Type getAccountType() { + return accountType; + } + + @Override + public long getDomainId() { + return domainId; + } + + @Override + public String getDomainUuid() { + return domainUuid; + } + + @Override + public String getDomainName() { + return domainName; + } + + @Override + public String getDomainPath() { + return domainPath; + } + + public long getProjectId() { + return projectId; + } + + @Override + public String getProjectUuid() { + return projectUuid; + } + + @Override + public String getProjectName() { + return projectName; + } + + public Long getDataCenterId() { + return dataCenterId; + } + + public String getDataCenterUuid() { + return dataCenterUuid; + } + + public String getDataCenterName() { + return dataCenterName; + } + + public Long getVolumeId() { + return volumeId; + } + + public String getVolumeUuid() { + return volumeUuid; + } + + public String getVolumeName() { + return volumeName; + } + + public Volume.Type getVolumeType() { + return volumeType; + } + + public Long getVolumeSize() { + return volumeSize; + } + + public Long getStoreId() { + return storeId; + } + + public String getStoreUuid() { + return storeUuid; + } + + public String getStoreName() { + return storeName; + } + + public DataStoreRole getStoreRole() { + return storeRole; + } + + public ObjectInDataStoreStateMachine.State getStoreState() { + return storeState; + } + + public VMTemplateStorageResourceAssoc.Status getDownloadState() { + return downloadState; + } + + public int getDownloadPercent() { + return downloadPercent; + } + + public String getErrorString() { + return errorString; + } + + public long getStoreSize() { + return storeSize; + } + + public Date getCreatedOnStore() { + return createdOnStore; + } + + public String getSnapshotStorePair() { + return snapshotStorePair; + } + + @Override + public Class getEntityType() { + return Snapshot.class; + } +} diff --git a/server/src/main/java/com/cloud/event/ActionEventUtils.java b/server/src/main/java/com/cloud/event/ActionEventUtils.java index c9cf933b0109..36461d20e421 100644 --- a/server/src/main/java/com/cloud/event/ActionEventUtils.java +++ b/server/src/main/java/com/cloud/event/ActionEventUtils.java @@ -318,7 +318,6 @@ private static Ternary updateParentResourceCases(Ternary> typeParentMethodMap = new HashMap<>(); - typeParentMethodMap.put(ApiCommandResourceType.Snapshot.toString(), new Pair<>(ApiCommandResourceType.Volume, "getVolumeId")); typeParentMethodMap.put(ApiCommandResourceType.VmSnapshot.toString(), new Pair<>(ApiCommandResourceType.VirtualMachine, "getVmId")); if (!typeParentMethodMap.containsKey(details.third())) { return details; diff --git a/server/src/main/java/com/cloud/server/ManagementServerImpl.java b/server/src/main/java/com/cloud/server/ManagementServerImpl.java index 3620e52d5476..02d958a15181 100644 --- a/server/src/main/java/com/cloud/server/ManagementServerImpl.java +++ b/server/src/main/java/com/cloud/server/ManagementServerImpl.java @@ -469,6 +469,7 @@ import org.apache.cloudstack.api.command.user.securitygroup.RevokeSecurityGroupIngressCmd; import org.apache.cloudstack.api.command.user.securitygroup.UpdateSecurityGroupCmd; import org.apache.cloudstack.api.command.user.snapshot.ArchiveSnapshotCmd; +import org.apache.cloudstack.api.command.user.snapshot.CopySnapshotCmd; import org.apache.cloudstack.api.command.user.snapshot.CreateSnapshotCmd; import org.apache.cloudstack.api.command.user.snapshot.CreateSnapshotFromVMSnapshotCmd; import org.apache.cloudstack.api.command.user.snapshot.CreateSnapshotPolicyCmd; @@ -3619,6 +3620,7 @@ public List> getCommands() { cmdList.add(UpdateSecurityGroupCmd.class); cmdList.add(CreateSnapshotCmd.class); cmdList.add(CreateSnapshotFromVMSnapshotCmd.class); + cmdList.add(CopySnapshotCmd.class); cmdList.add(DeleteSnapshotCmd.class); cmdList.add(ArchiveSnapshotCmd.class); cmdList.add(CreateSnapshotPolicyCmd.class); diff --git a/server/src/main/java/com/cloud/storage/CreateSnapshotPayload.java b/server/src/main/java/com/cloud/storage/CreateSnapshotPayload.java index b0bb9eafb2eb..868f785bdc2b 100644 --- a/server/src/main/java/com/cloud/storage/CreateSnapshotPayload.java +++ b/server/src/main/java/com/cloud/storage/CreateSnapshotPayload.java @@ -16,6 +16,8 @@ // under the License. package com.cloud.storage; +import java.util.List; + import com.cloud.user.Account; public class CreateSnapshotPayload { @@ -25,6 +27,7 @@ public class CreateSnapshotPayload { private boolean quiescevm; private Snapshot.LocationType locationType; private boolean asyncBackup; + private List zoneIds; public Long getSnapshotPolicyId() { return snapshotPolicyId; @@ -67,4 +70,12 @@ public void setAsyncBackup(boolean asyncBackup) { public boolean getAsyncBackup() { return this.asyncBackup; } + + public List getZoneIds() { + return zoneIds; + } + + public void setZoneIds(List zoneIds) { + this.zoneIds = zoneIds; + } } diff --git a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java index 56b028720d2c..0618a0f51047 100644 --- a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java @@ -1316,37 +1316,34 @@ public void cleanupStorage(boolean recurring) { //destroy snapshots in destroying state in snapshot_store_ref List ssSnapshots = _snapshotStoreDao.listByState(ObjectInDataStoreStateMachine.State.Destroying); - for (SnapshotDataStoreVO ssSnapshotVO : ssSnapshots) { + for (SnapshotDataStoreVO snapshotDataStoreVO : ssSnapshots) { String snapshotUuid = null; SnapshotVO snapshot = null; - + final String storeRole = snapshotDataStoreVO.getRole().toString().toLowerCase(); if (s_logger.isDebugEnabled()) { - snapshot = _snapshotDao.findById(ssSnapshotVO.getSnapshotId()); + snapshot = _snapshotDao.findById(snapshotDataStoreVO.getSnapshotId()); if (snapshot == null) { - s_logger.warn(String.format("Did not find snapshot [%s] in destroying state; therefore, it cannot be destroyed.", ssSnapshotVO.getSnapshotId())); + s_logger.warn(String.format("Did not find snapshot [ID: %d] for which store reference is in destroying state; therefore, it cannot be destroyed.", snapshotDataStoreVO.getSnapshotId())); continue; } - snapshotUuid = snapshot.getUuid(); } try { if (s_logger.isDebugEnabled()) { - s_logger.debug(String.format("Verifying if snapshot [%s] is in destroying state in any image data store.", snapshotUuid)); + s_logger.debug(String.format("Verifying if snapshot [%s] is in destroying state in %s data store ID: %d.", snapshotUuid, storeRole, snapshotDataStoreVO.getDataStoreId())); } - - SnapshotInfo snapshotInfo = snapshotFactory.getSnapshot(ssSnapshotVO.getSnapshotId(), DataStoreRole.Image); - + SnapshotInfo snapshotInfo = snapshotFactory.getSnapshot(snapshotDataStoreVO.getSnapshotId(), snapshotDataStoreVO.getDataStoreId(), snapshotDataStoreVO.getRole()); if (snapshotInfo != null) { if (s_logger.isDebugEnabled()) { - s_logger.debug(String.format("Snapshot [%s] in destroying state found in image data store [%s]; therefore, it will be destroyed.", snapshotUuid, snapshotInfo.getDataStore().getUuid())); + s_logger.debug(String.format("Snapshot [%s] in destroying state found in %s data store [%s]; therefore, it will be destroyed.", snapshotUuid, storeRole, snapshotInfo.getDataStore().getUuid())); } _snapshotService.deleteSnapshot(snapshotInfo); } else if (s_logger.isDebugEnabled()) { - s_logger.debug(String.format("Did not find snapshot [%s] in destroying state in any image data store.", snapshotUuid)); + s_logger.debug(String.format("Did not find snapshot [%s] in destroying state in %s data store ID: %d.", snapshotUuid, storeRole, snapshotDataStoreVO.getDataStoreId())); } } catch (Exception e) { - s_logger.error(String.format("Failed to delete snapshot [%s] from storage due to: [%s].", ssSnapshotVO.getSnapshotId(), e.getMessage())); + s_logger.error(String.format("Failed to delete snapshot [%s] from storage due to: [%s].", snapshotDataStoreVO.getSnapshotId(), e.getMessage())); if (s_logger.isDebugEnabled()) { s_logger.debug(String.format("Failed to delete snapshot [%s] from storage.", snapshotUuid), e); } @@ -1668,7 +1665,10 @@ public void cleanupSecondaryStorage(boolean recurring) { s_logger.debug("Deleting snapshot store DB entry: " + destroyedSnapshotStoreVO); } - _snapshotDao.remove(destroyedSnapshotStoreVO.getSnapshotId()); + List imageStoreRefs = _snapshotStoreDao.listBySnapshot(destroyedSnapshotStoreVO.getSnapshotId(), DataStoreRole.Image); + if (imageStoreRefs.size() <= 1) { + _snapshotDao.remove(destroyedSnapshotStoreVO.getSnapshotId()); + } SnapshotDataStoreVO snapshotOnPrimary = _snapshotStoreDao.findDestroyedReferenceBySnapshot(destroyedSnapshotStoreVO.getSnapshotId(), DataStoreRole.Primary); if (snapshotOnPrimary != null) { if (s_logger.isDebugEnabled()) { diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index 3fba19947235..7bfe8a0818b4 100644 --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@ -31,11 +31,11 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; import javax.inject.Inject; -import com.cloud.domain.dao.DomainDao; -import org.apache.cloudstack.api.ApiConstants.IoDriverPolicy; +import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.InternalIdentity; import org.apache.cloudstack.api.ServerApiException; @@ -66,6 +66,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreDriver; import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreInfo; import org.apache.cloudstack.engine.subsystem.api.storage.Scope; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; import org.apache.cloudstack.engine.subsystem.api.storage.StoragePoolAllocator; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; @@ -85,7 +86,9 @@ import org.apache.cloudstack.framework.jobs.impl.VmWorkJobVO; import org.apache.cloudstack.jobs.JobInfo; import org.apache.cloudstack.resourcedetail.DiskOfferingDetailVO; +import org.apache.cloudstack.resourcedetail.SnapshotPolicyDetailVO; import org.apache.cloudstack.resourcedetail.dao.DiskOfferingDetailsDao; +import org.apache.cloudstack.resourcedetail.dao.SnapshotPolicyDetailsDao; import org.apache.cloudstack.snapshot.SnapshotHelper; import org.apache.cloudstack.storage.command.AttachAnswer; import org.apache.cloudstack.storage.command.AttachCommand; @@ -95,8 +98,8 @@ import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; -import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreDao; import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO; import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity; @@ -133,6 +136,7 @@ import com.cloud.dc.Pod; import com.cloud.dc.dao.DataCenterDao; import com.cloud.domain.Domain; +import com.cloud.domain.dao.DomainDao; import com.cloud.event.ActionEvent; import com.cloud.event.EventTypes; import com.cloud.event.UsageEventUtils; @@ -260,6 +264,8 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic @Inject private SnapshotDao _snapshotDao; @Inject + private SnapshotPolicyDetailsDao snapshotPolicyDetailsDao; + @Inject private SnapshotDataStoreDao _snapshotDataStoreDao; @Inject private ServiceOfferingDetailsDao _serviceOfferingDetailsDao; @@ -818,7 +824,7 @@ public VolumeVO allocVolume(CreateVolumeCmd cmd) throws ResourceAllocationExcept throw new InvalidParameterValueException("Snapshot id=" + snapshotId + " is not in " + Snapshot.State.BackedUp + " state yet and can't be used for volume creation"); } - SnapshotDataStoreVO snapshotStore = _snapshotDataStoreDao.findBySnapshot(snapshotId, DataStoreRole.Primary); + SnapshotDataStoreVO snapshotStore = _snapshotDataStoreDao.findOneBySnapshotAndDatastoreRole(snapshotId, DataStoreRole.Primary); if (snapshotStore != null) { StoragePoolVO storagePoolVO = _storagePoolDao.findById(snapshotStore.getDataStoreId()); if (storagePoolVO.getPoolType() == Storage.StoragePoolType.PowerFlex) { @@ -835,7 +841,7 @@ public VolumeVO allocVolume(CreateVolumeCmd cmd) throws ResourceAllocationExcept if (zoneId == null) { // if zoneId is not provided, we default to create volume in the same zone as the snapshot zone. - zoneId = snapshotCheck.getDataCenterId(); + zoneId = parentVolume.getDataCenterId(); } if (diskOffering == null) { // Pure snapshot is being used to create volume. @@ -866,7 +872,9 @@ public VolumeVO allocVolume(CreateVolumeCmd cmd) throws ResourceAllocationExcept if (vm == null || vm.getType() != VirtualMachine.Type.User) { throw new InvalidParameterValueException("Please specify a valid User VM."); } - + if (vm.getDataCenterId() != zoneId) { + throw new InvalidParameterValueException("The specified zone is different than zone of the VM"); + } // Check that the VM is in the correct state if (vm.getState() != State.Running && vm.getState() != State.Stopped) { throw new InvalidParameterValueException("Please specify a VM that is either running or stopped."); @@ -3402,22 +3410,39 @@ protected Volume liveMigrateVolume(Volume volume, StoragePool destPool) throws S @Override @ActionEvent(eventType = EventTypes.EVENT_SNAPSHOT_CREATE, eventDescription = "taking snapshot", async = true) - public Snapshot takeSnapshot(Long volumeId, Long policyId, Long snapshotId, Account account, boolean quiescevm, Snapshot.LocationType locationType, boolean asyncBackup, Map tags) + public Snapshot takeSnapshot(Long volumeId, Long policyId, Long snapshotId, Account account, boolean quiescevm, + Snapshot.LocationType locationType, boolean asyncBackup, Map tags, List zoneIds) throws ResourceAllocationException { - final Snapshot snapshot = takeSnapshotInternal(volumeId, policyId, snapshotId, account, quiescevm, locationType, asyncBackup); + final Snapshot snapshot = takeSnapshotInternal(volumeId, policyId, snapshotId, account, quiescevm, locationType, asyncBackup, zoneIds); if (snapshot != null && MapUtils.isNotEmpty(tags)) { taggedResourceService.createTags(Collections.singletonList(snapshot.getUuid()), ResourceTag.ResourceObjectType.Snapshot, tags, null); } return snapshot; } - private Snapshot takeSnapshotInternal(Long volumeId, Long policyId, Long snapshotId, Account account, boolean quiescevm, Snapshot.LocationType locationType, boolean asyncBackup) + private Snapshot takeSnapshotInternal(Long volumeId, Long policyId, Long snapshotId, Account account, + boolean quiescevm, Snapshot.LocationType locationType, boolean asyncBackup, List zoneIds) throws ResourceAllocationException { Account caller = CallContext.current().getCallingAccount(); VolumeInfo volume = volFactory.getVolume(volumeId); if (volume == null) { throw new InvalidParameterValueException("Creating snapshot failed due to volume:" + volumeId + " doesn't exist"); } + if (policyId != null && policyId > 0) { + if (CollectionUtils.isNotEmpty(zoneIds)) { + throw new InvalidParameterValueException(String.format("%s can not be specified for snapshots linked with snapshot policy", ApiConstants.ZONE_ID_LIST)); + } + List details = snapshotPolicyDetailsDao.findDetails(policyId, ApiConstants.ZONE_ID); + zoneIds = details.stream().map(d -> Long.valueOf(d.getValue())).collect(Collectors.toList()); + } + if (CollectionUtils.isNotEmpty(zoneIds)) { + for (Long destZoneId : zoneIds) { + DataCenterVO dstZone = _dcDao.findById(destZoneId); + if (dstZone == null) { + throw new InvalidParameterValueException("Please specify a valid destination zone."); + } + } + } _accountMgr.checkAccess(caller, null, true, volume); @@ -3446,13 +3471,15 @@ private Snapshot takeSnapshotInternal(Long volumeId, Long policyId, Long snapsho VmWorkJobVO placeHolder = null; placeHolder = createPlaceHolderWork(vm.getId()); try { - return orchestrateTakeVolumeSnapshot(volumeId, policyId, snapshotId, account, quiescevm, locationType, asyncBackup); + return orchestrateTakeVolumeSnapshot(volumeId, policyId, snapshotId, account, quiescevm, + locationType, asyncBackup, zoneIds); } finally { _workJobDao.expunge(placeHolder.getId()); } } else { - Outcome outcome = takeVolumeSnapshotThroughJobQueue(vm.getId(), volumeId, policyId, snapshotId, account.getId(), quiescevm, locationType, asyncBackup); + Outcome outcome = takeVolumeSnapshotThroughJobQueue(vm.getId(), volumeId, policyId, + snapshotId, account.getId(), quiescevm, locationType, asyncBackup, zoneIds); try { outcome.get(); @@ -3482,12 +3509,16 @@ private Snapshot takeSnapshotInternal(Long volumeId, Long policyId, Long snapsho payload.setAccount(account); payload.setQuiescevm(quiescevm); payload.setAsyncBackup(asyncBackup); + if (CollectionUtils.isNotEmpty(zoneIds)) { + payload.setZoneIds(zoneIds); + } volume.addPayload(payload); return volService.takeSnapshot(volume); } } - private Snapshot orchestrateTakeVolumeSnapshot(Long volumeId, Long policyId, Long snapshotId, Account account, boolean quiescevm, Snapshot.LocationType locationType, boolean asyncBackup) + private Snapshot orchestrateTakeVolumeSnapshot(Long volumeId, Long policyId, Long snapshotId, Account account, + boolean quiescevm, Snapshot.LocationType locationType, boolean asyncBackup, List zoneIds) throws ResourceAllocationException { VolumeInfo volume = volFactory.getVolume(volumeId); @@ -3514,6 +3545,9 @@ private Snapshot orchestrateTakeVolumeSnapshot(Long volumeId, Long policyId, Lon payload.setQuiescevm(quiescevm); payload.setLocationType(locationType); payload.setAsyncBackup(asyncBackup); + if (CollectionUtils.isNotEmpty(zoneIds)) { + payload.setZoneIds(zoneIds); + } volume.addPayload(payload); return volService.takeSnapshot(volume); @@ -3529,7 +3563,7 @@ private boolean isOperationSupported(VMTemplateVO template, UserVmVO userVm) { @Override @ActionEvent(eventType = EventTypes.EVENT_SNAPSHOT_CREATE, eventDescription = "allocating snapshot", create = true) - public Snapshot allocSnapshot(Long volumeId, Long policyId, String snapshotName, Snapshot.LocationType locationType) throws ResourceAllocationException { + public Snapshot allocSnapshot(Long volumeId, Long policyId, String snapshotName, Snapshot.LocationType locationType, List zoneIds) throws ResourceAllocationException { Account caller = CallContext.current().getCallingAccount(); VolumeInfo volume = volFactory.getVolume(volumeId); @@ -3538,7 +3572,7 @@ public Snapshot allocSnapshot(Long volumeId, Long policyId, String snapshotName, } DataCenter zone = _dcDao.findById(volume.getDataCenterId()); if (zone == null) { - throw new InvalidParameterValueException("Can't find zone by id " + volume.getDataCenterId()); + throw new InvalidParameterValueException(String.format("Can't find zone for the volume ID: %s", volume.getUuid())); } if (Grouping.AllocationState.Disabled == zone.getAllocationState() && !_accountMgr.isRootAdmin(caller.getId())) { @@ -3552,7 +3586,6 @@ public Snapshot allocSnapshot(Long volumeId, Long policyId, String snapshotName, if (ImageFormat.DIR.equals(volume.getFormat())) { throw new InvalidParameterValueException("Snapshot not supported for volume:" + volumeId); } - if (volume.getTemplateId() != null) { VMTemplateVO template = _templateDao.findById(volume.getTemplateId()); Long instanceId = volume.getInstanceId(); @@ -3580,7 +3613,35 @@ public Snapshot allocSnapshot(Long volumeId, Long policyId, String snapshotName, throw new InvalidParameterValueException("VolumeId: " + volumeId + " please attach this volume to a VM before create snapshot for it"); } - return snapshotMgr.allocSnapshot(volumeId, policyId, snapshotName, locationType, false); + if (CollectionUtils.isNotEmpty(zoneIds)) { + if (policyId != null && policyId > 0) { + throw new InvalidParameterValueException(String.format("%s parameter can not be specified with %s parameter", ApiConstants.ZONE_ID_LIST, ApiConstants.POLICY_ID)); + } + if (Snapshot.LocationType.PRIMARY.equals(locationType)) { + throw new InvalidParameterValueException(String.format("%s cannot be specified with snapshot %s as %s", ApiConstants.ZONE_ID_LIST, ApiConstants.LOCATION_TYPE, Snapshot.LocationType.PRIMARY)); + } + if (Boolean.FALSE.equals(SnapshotInfo.BackupSnapshotAfterTakingSnapshot.value())) { + throw new InvalidParameterValueException("Backing up of snapshot has been disabled. Snapshot can not be taken for multiple zones"); + } + if (DataCenter.Type.Edge.equals(zone.getType())) { + throw new InvalidParameterValueException("Backing up of snapshot is not supported by the zone of the volume. Snapshot can not be taken for multiple zones"); + } + for (Long zoneId : zoneIds) { + DataCenter dataCenter = _dcDao.findById(zoneId); + if (dataCenter == null) { + throw new InvalidParameterValueException("Unable to find the specified zone"); + } + if (Grouping.AllocationState.Disabled.equals(dataCenter.getAllocationState()) && !_accountMgr.isRootAdmin(caller.getId())) { + throw new PermissionDeniedException("Cannot perform this operation, Zone is currently disabled: " + dataCenter.getName()); + } + if (DataCenter.Type.Edge.equals(dataCenter.getType())) { + throw new InvalidParameterValueException("Snapshot functionality is not supported on zone %s"); + } + } + } + + + return snapshotMgr.allocSnapshot(volumeId, policyId, snapshotName, locationType, false, zoneIds); } @Override @@ -3636,7 +3697,7 @@ public Snapshot allocSnapshotForVm(Long vmId, Long volumeId, String snapshotName throw new InvalidParameterValueException("Cannot perform this operation, unsupported on storage pool type " + storagePool.getPoolType()); } - return snapshotMgr.allocSnapshot(volumeId, Snapshot.MANUAL_POLICY_ID, snapshotName, null, true); + return snapshotMgr.allocSnapshot(volumeId, Snapshot.MANUAL_POLICY_ID, snapshotName, null, true, null); } @Override @@ -4336,7 +4397,7 @@ private String getIoPolicy(UserVmVO vm, long poolId) { String ioPolicy = null; if (vm.getHypervisorType() == HypervisorType.KVM && vm.getDetails() != null && vm.getDetail(VmDetailConstants.IO_POLICY) != null) { ioPolicy = vm.getDetail(VmDetailConstants.IO_POLICY); - if (IoDriverPolicy.STORAGE_SPECIFIC.toString().equals(ioPolicy)) { + if (ApiConstants.IoDriverPolicy.STORAGE_SPECIFIC.toString().equals(ioPolicy)) { String storageIoPolicyDriver = StorageManager.STORAGE_POOL_IO_POLICY.valueIn(poolId); ioPolicy = storageIoPolicyDriver != null ? storageIoPolicyDriver : null; } @@ -4670,7 +4731,7 @@ private Outcome migrateVolumeThroughJobQueue(VMInstanceVO vm, VolumeVO v } public Outcome takeVolumeSnapshotThroughJobQueue(final Long vmId, final Long volumeId, final Long policyId, final Long snapshotId, final Long accountId, final boolean quiesceVm, - final Snapshot.LocationType locationType, final boolean asyncBackup) { + final Snapshot.LocationType locationType, final boolean asyncBackup, final List zoneIds) { final CallContext context = CallContext.current(); final User callingUser = context.getCallingUser(); @@ -4692,7 +4753,7 @@ public Outcome takeVolumeSnapshotThroughJobQueue(final Long vmId, fina // save work context info (there are some duplications) VmWorkTakeVolumeSnapshot workInfo = new VmWorkTakeVolumeSnapshot(callingUser.getId(), accountId != null ? accountId : callingAccount.getId(), vm.getId(), - VolumeApiServiceImpl.VM_WORK_JOB_HANDLER, volumeId, policyId, snapshotId, quiesceVm, locationType, asyncBackup); + VolumeApiServiceImpl.VM_WORK_JOB_HANDLER, volumeId, policyId, snapshotId, quiesceVm, locationType, asyncBackup, zoneIds); workJob.setCmdInfo(VmWorkSerializer.serialize(workInfo)); _jobMgr.submitAsyncJob(workJob, VmWorkConstants.VM_WORK_QUEUE, vm.getId()); @@ -4742,7 +4803,8 @@ private Pair orchestrateMigrateVolume(VmWorkMigrateVolum @ReflectionUse private Pair orchestrateTakeVolumeSnapshot(VmWorkTakeVolumeSnapshot work) throws Exception { Account account = _accountDao.findById(work.getAccountId()); - orchestrateTakeVolumeSnapshot(work.getVolumeId(), work.getPolicyId(), work.getSnapshotId(), account, work.isQuiesceVm(), work.getLocationType(), work.isAsyncBackup()); + orchestrateTakeVolumeSnapshot(work.getVolumeId(), work.getPolicyId(), work.getSnapshotId(), account, + work.isQuiesceVm(), work.getLocationType(), work.isAsyncBackup(), work.getZoneIds()); return new Pair(JobInfo.Status.SUCCEEDED, _jobMgr.marshallResultObject(work.getSnapshotId())); } diff --git a/server/src/main/java/com/cloud/storage/download/DownloadListener.java b/server/src/main/java/com/cloud/storage/download/DownloadListener.java index 9f528195bfa3..7cd2e2a790a6 100644 --- a/server/src/main/java/com/cloud/storage/download/DownloadListener.java +++ b/server/src/main/java/com/cloud/storage/download/DownloadListener.java @@ -181,6 +181,8 @@ public void sendCommand(RequestType reqType) { DownloadProgressCommand dcmd = new DownloadProgressCommand(getCommand(), getJobId(), reqType); if (object.getType() == DataObjectType.VOLUME) { dcmd.setResourceType(ResourceType.VOLUME); + } else if (object.getType() == DataObjectType.SNAPSHOT) { + dcmd.setResourceType(ResourceType.SNAPSHOT); } _ssAgent.sendMessageAsync(dcmd, new UploadListener.Callback(_ssAgent.getId(), this)); } catch (Exception e) { diff --git a/server/src/main/java/com/cloud/storage/download/DownloadMonitor.java b/server/src/main/java/com/cloud/storage/download/DownloadMonitor.java index b93c982b51df..028a957ee335 100644 --- a/server/src/main/java/com/cloud/storage/download/DownloadMonitor.java +++ b/server/src/main/java/com/cloud/storage/download/DownloadMonitor.java @@ -32,4 +32,6 @@ public interface DownloadMonitor extends Manager { public void downloadVolumeToStorage(DataObject volume, AsyncCompletionCallback callback); + void downloadSnapshotToStorage(DataObject volume, AsyncCompletionCallback callback); + } diff --git a/server/src/main/java/com/cloud/storage/download/DownloadMonitorImpl.java b/server/src/main/java/com/cloud/storage/download/DownloadMonitorImpl.java index 1954cdea6879..90782dd934b9 100644 --- a/server/src/main/java/com/cloud/storage/download/DownloadMonitorImpl.java +++ b/server/src/main/java/com/cloud/storage/download/DownloadMonitorImpl.java @@ -25,9 +25,6 @@ import javax.inject.Inject; -import org.apache.log4j.Logger; -import org.springframework.stereotype.Component; - import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; @@ -39,17 +36,22 @@ import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType; import org.apache.cloudstack.storage.command.DownloadProgressCommand; import org.apache.cloudstack.storage.command.DownloadProgressCommand.RequestType; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreDao; import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO; +import org.apache.cloudstack.storage.to.SnapshotObjectTO; import org.apache.cloudstack.storage.to.TemplateObjectTO; import org.apache.cloudstack.storage.to.VolumeObjectTO; +import org.apache.log4j.Logger; +import org.springframework.stereotype.Component; import com.cloud.agent.AgentManager; import com.cloud.agent.api.storage.DownloadAnswer; -import com.cloud.utils.net.Proxy; import com.cloud.configuration.Config; +import com.cloud.storage.DataStoreRole; import com.cloud.storage.RegisterVolumePayload; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.VMTemplateStorageResourceAssoc.Status; @@ -58,6 +60,7 @@ import com.cloud.utils.component.ComponentContext; import com.cloud.utils.component.ManagerBase; import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.net.Proxy; @Component public class DownloadMonitorImpl extends ManagerBase implements DownloadMonitor { @@ -68,6 +71,8 @@ public class DownloadMonitorImpl extends ManagerBase implements DownloadMonitor @Inject private VolumeDataStoreDao _volumeStoreDao; @Inject + private SnapshotDataStoreDao snapshotDataStoreDao; + @Inject private AgentManager _agentMgr; @Inject private ConfigurationDao _configDao; @@ -115,6 +120,12 @@ public boolean isTemplateUpdateable(Long templateId, Long storeId) { return (downloadsInProgress.size() == 0); } + public boolean isSnapshotUpdateable(Long snapshotId, Long storeId) { + List downloadsInProgress = + snapshotDataStoreDao.listBySnasphotStoreDownloadStatus(snapshotId, storeId, Status.DOWNLOAD_IN_PROGRESS, Status.DOWNLOADED); + return downloadsInProgress.isEmpty(); + } + private void initiateTemplateDownload(DataObject template, AsyncCompletionCallback callback) { boolean downloadJobExists = false; TemplateDataStoreVO vmTemplateStore; @@ -169,6 +180,63 @@ private void initiateTemplateDownload(DataObject template, AsyncCompletionCallba } } + private void initiateSnapshotDownload(DataObject snapshot, AsyncCompletionCallback callback) { + boolean downloadJobExists = false; + DataStore store = snapshot.getDataStore(); + + SnapshotDataStoreVO snapshotStore = snapshotDataStoreDao.findByStoreSnapshot(DataStoreRole.Image, store.getId(), snapshot.getId()); + if (snapshotStore == null) { + snapshotStore = + new SnapshotDataStoreVO(store.getId(), snapshot.getId()); + snapshotStore.setLastUpdated(new Date()); + snapshotStore.setDownloadPercent(0); + snapshotStore.setDownloadState(Status.NOT_DOWNLOADED); + snapshotStore.setLocalDownloadPath(null); + snapshotStore.setErrorString(null); + snapshotStore.setJobId("jobid0000"); + snapshotStore.setRole(store.getRole()); + snapshotStore = snapshotDataStoreDao.persist(snapshotStore); + } else if ((snapshotStore.getJobId() != null) && (snapshotStore.getJobId().length() > 2)) { + downloadJobExists = true; + } + + Long maxSizeInBytes = getMaxSnapshotSizeInBytes(); + if (snapshotStore != null) { + start(); + DownloadCommand dcmd = new DownloadCommand((SnapshotObjectTO)(snapshot.getTO()), maxSizeInBytes, snapshot.getUri()); + dcmd.setProxy(getHttpProxy()); + if (downloadJobExists) { + dcmd = new DownloadProgressCommand(dcmd, snapshotStore.getJobId(), RequestType.GET_OR_RESTART); + dcmd.setResourceType(ResourceType.SNAPSHOT); + } + EndPoint ep = _epSelector.select(snapshot); + if (ep == null) { + String errMsg = "There is no secondary storage VM for downloading snapshot to image store " + store.getName(); + LOGGER.warn(errMsg); + throw new CloudRuntimeException(errMsg); + } + DownloadListener dl = new DownloadListener(ep, store, snapshot, _timer, this, dcmd, callback); + ComponentContext.inject(dl); // initialize those auto-wired field in download listener. + if (downloadJobExists) { + // due to handling existing download job issues, we still keep + // downloadState in template_store_ref to avoid big change in + // DownloadListener to use + // new ObjectInDataStore.State transition. TODO: fix this later + // to be able to remove downloadState from template_store_ref. + LOGGER.info("found existing download job"); + dl.setCurrState(snapshotStore.getDownloadState()); + } + + try { + ep.sendMessageAsync(dcmd, new UploadListener.Callback(ep.getId(), dl)); + } catch (Exception e) { + LOGGER.warn("Unable to start /resume download of snapshot " + snapshot.getId() + " to " + store.getName(), e); + dl.setDisconnected(); + dl.scheduleStatusCheck(RequestType.GET_OR_RESTART); + } + } + } + @Override public void downloadTemplateToStorage(DataObject template, AsyncCompletionCallback callback) { if(template != null) { @@ -245,6 +313,26 @@ public void downloadVolumeToStorage(DataObject volume, AsyncCompletionCallback callback) { + long snapshotId = snapshot.getId(); + DataStore store = snapshot.getDataStore(); + if (isSnapshotUpdateable(snapshotId, store.getId())) { + if (snapshot.getUri() != null) { + initiateSnapshotDownload(snapshot, callback); + } else { + LOGGER.info("Snapshot url is null, cannot download"); + DownloadAnswer ans = new DownloadAnswer("Snapshot url is null", Status.UNKNOWN); + callback.complete(ans); + } + } else { + LOGGER.info("Snapshot download is already in progress or already downloaded"); + DownloadAnswer ans = + new DownloadAnswer("Snapshot download is already in progress or already downloaded", Status.UNKNOWN); + callback.complete(ans); + } + } + private Long getMaxTemplateSizeInBytes() { try { return Long.parseLong(_configDao.getValue("max.template.iso.size")) * 1024L * 1024L * 1024L; @@ -261,6 +349,14 @@ private Long getMaxVolumeSizeInBytes() { } } + private Long getMaxSnapshotSizeInBytes() { + try { + return Long.parseLong(_configDao.getValue("storage.max.volume.upload.size")) * 1024L * 1024L * 1024L; + } catch (NumberFormatException e) { + return null; + } + } + private Proxy getHttpProxy() { if (_proxy == null) { return null; diff --git a/server/src/main/java/com/cloud/storage/snapshot/SnapshotManager.java b/server/src/main/java/com/cloud/storage/snapshot/SnapshotManager.java index 7219e0dbb6f0..dd63371b888d 100644 --- a/server/src/main/java/com/cloud/storage/snapshot/SnapshotManager.java +++ b/server/src/main/java/com/cloud/storage/snapshot/SnapshotManager.java @@ -70,8 +70,6 @@ public interface SnapshotManager extends Configurable { */ boolean deleteSnapshotDirsForAccount(long accountId); - String getSecondaryStorageURL(SnapshotVO snapshot); - //void deleteSnapshotsDirForVolume(String secondaryStoragePoolUrl, Long dcId, Long accountId, Long volumeId); boolean canOperateOnVolume(Volume volume); diff --git a/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java b/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java index bd8811b2a157..922ebd529186 100755 --- a/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java @@ -19,25 +19,32 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Date; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.TimeZone; +import java.util.concurrent.ExecutionException; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import javax.inject.Inject; import javax.naming.ConfigurationException; +import org.apache.cloudstack.acl.SecurityChecker; import org.apache.cloudstack.annotation.AnnotationService; import org.apache.cloudstack.annotation.dao.AnnotationDao; import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.command.user.snapshot.CopySnapshotCmd; import org.apache.cloudstack.api.command.user.snapshot.CreateSnapshotPolicyCmd; import org.apache.cloudstack.api.command.user.snapshot.DeleteSnapshotPoliciesCmd; import org.apache.cloudstack.api.command.user.snapshot.ListSnapshotPoliciesCmd; import org.apache.cloudstack.api.command.user.snapshot.ListSnapshotsCmd; import org.apache.cloudstack.api.command.user.snapshot.UpdateSnapshotPolicyCmd; import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; @@ -45,6 +52,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotResult; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotService; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotStrategy; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotStrategy.SnapshotOperation; @@ -52,10 +60,13 @@ import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope; +import org.apache.cloudstack.framework.async.AsyncCallFuture; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.managed.context.ManagedContextRunnable; +import org.apache.cloudstack.resourcedetail.SnapshotPolicyDetailVO; +import org.apache.cloudstack.resourcedetail.dao.SnapshotPolicyDetailsDao; import org.apache.cloudstack.snapshot.SnapshotHelper; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; @@ -64,6 +75,7 @@ import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.builder.ReflectionToStringBuilder; import org.apache.commons.lang3.builder.ToStringStyle; import org.apache.log4j.Logger; @@ -91,9 +103,11 @@ import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.PermissionDeniedException; import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; import com.cloud.exception.StorageUnavailableException; import com.cloud.host.HostVO; import com.cloud.hypervisor.Hypervisor.HypervisorType; +import com.cloud.org.Grouping; import com.cloud.projects.Project.ListProjectResourcesCriteria; import com.cloud.resource.ResourceManager; import com.cloud.server.ResourceTag.ResourceObjectType; @@ -111,6 +125,7 @@ import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.StorageManager; import com.cloud.storage.StoragePool; +import com.cloud.storage.VMTemplateStorageResourceAssoc; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.Volume; import com.cloud.storage.VolumeVO; @@ -118,6 +133,7 @@ import com.cloud.storage.dao.SnapshotDao; import com.cloud.storage.dao.SnapshotPolicyDao; import com.cloud.storage.dao.SnapshotScheduleDao; +import com.cloud.storage.dao.SnapshotZoneDao; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VolumeDao; import com.cloud.storage.template.TemplateConstants; @@ -171,7 +187,9 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement @Inject PrimaryDataStoreDao _storagePoolDao; @Inject - SnapshotPolicyDao _snapshotPolicyDao = null; + SnapshotPolicyDao _snapshotPolicyDao; + @Inject + SnapshotPolicyDetailsDao snapshotPolicyDetailsDao; @Inject SnapshotScheduleDao _snapshotScheduleDao; @Inject @@ -221,6 +239,8 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement protected SnapshotHelper snapshotHelper; @Inject DataCenterDao dataCenterDao; + @Inject + SnapshotZoneDao snapshotZoneDao; private int _totalRetries; private int _pauseInterval; @@ -228,6 +248,17 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement private ScheduledExecutorService backupSnapshotExecutor; + protected DataStore getSnapshotZoneImageStore(long snapshotId, long zoneId) { + List snapshotImageStoreList = _snapshotStoreDao.listReadyBySnapshot(snapshotId, DataStoreRole.Image); + for (SnapshotDataStoreVO ref : snapshotImageStoreList) { + Long entryZoneId = dataStoreMgr.getStoreZoneId(ref.getDataStoreId(), ref.getRole()); + if (entryZoneId != null && entryZoneId.equals(zoneId)) { + return dataStoreMgr.getDataStore(ref.getDataStoreId(), ref.getRole()); + } + } + return null; + } + protected boolean isBackupSnapshotToSecondaryForZone(long zoneId) { if (Boolean.FALSE.equals(SnapshotInfo.BackupSnapshotAfterTakingSnapshot.value())) { return false; @@ -334,7 +365,7 @@ public Snapshot revertSnapshot(Long snapshotId) { DataStoreRole dataStoreRole = snapshotHelper.getDataStoreRole(snapshot); - SnapshotInfo snapshotInfo = snapshotFactory.getSnapshot(snapshotId, dataStoreRole); + SnapshotInfo snapshotInfo = snapshotFactory.getSnapshotWithRoleAndZone(snapshotId, dataStoreRole, volume.getDataCenterId()); if (snapshotInfo == null) { throw new CloudRuntimeException(String.format("snapshot %s [%s] does not exists in data store", snapshot.getName(), snapshot.getUuid())); @@ -407,7 +438,7 @@ public Snapshot createSnapshot(Long volumeId, Long policyId, Long snapshotId, Ac // does the caller have the authority to act on this volume _accountMgr.checkAccess(CallContext.current().getCallingAccount(), null, true, volume); - SnapshotInfo snapshot = snapshotFactory.getSnapshot(snapshotId, DataStoreRole.Primary); + SnapshotInfo snapshot = snapshotFactory.getSnapshotOnPrimaryStore(snapshotId); if (snapshot == null) { s_logger.debug("Failed to create snapshot"); throw new CloudRuntimeException("Failed to create snapshot"); @@ -432,7 +463,7 @@ public Snapshot createSnapshot(Long volumeId, Long policyId, Long snapshotId, Ac @Override public Snapshot archiveSnapshot(Long snapshotId) { - SnapshotInfo snapshotOnPrimary = snapshotFactory.getSnapshot(snapshotId, DataStoreRole.Primary); + SnapshotInfo snapshotOnPrimary = snapshotFactory.getSnapshotOnPrimaryStore(snapshotId); if (snapshotOnPrimary == null || !snapshotOnPrimary.getStatus().equals(ObjectInDataStoreStateMachine.State.Ready)) { throw new CloudRuntimeException("Can only archive snapshots present on primary storage. " + "Cannot find snapshot " + snapshotId + " on primary storage"); @@ -576,7 +607,7 @@ private void postCreateRecurringSnapshotForPolicy(long userId, long volumeId, lo if (policy != null) { s_logger.debug("Max snaps: " + policy.getMaxSnaps() + " exceeded for snapshot policy with Id: " + policyId + ". Deleting oldest snapshot: " + oldSnapId); } - if (deleteSnapshot(oldSnapId)) { + if (deleteSnapshot(oldSnapId, null)) { //log Snapshot delete event ActionEventUtils.onCompletedActionEvent(User.UID_SYSTEM, oldestSnapshot.getAccountId(), EventVO.LEVEL_INFO, EventTypes.EVENT_SNAPSHOT_DELETE, "Successfully deleted oldest snapshot: " + oldSnapId, oldSnapId, ApiCommandResourceType.Snapshot.toString(), 0); @@ -585,14 +616,42 @@ private void postCreateRecurringSnapshotForPolicy(long userId, long volumeId, lo } } + protected Pair, List> getStoreRefsAndZonesForSnapshotDelete(long snapshotId, Long zoneId) { + List snapshotStoreRefs = new ArrayList<>(); + List allSnapshotStoreRefs = _snapshotStoreDao.findBySnapshotId(snapshotId); + List zoneIds = new ArrayList<>(); + if (zoneId != null) { + DataCenterVO zone = dataCenterDao.findById(zoneId); + if (zone == null) { + throw new InvalidParameterValueException("Unable to find a zone with the specified id"); + } + for (SnapshotDataStoreVO snapshotStore : allSnapshotStoreRefs) { + Long entryZoneId = dataStoreMgr.getStoreZoneId(snapshotStore.getDataStoreId(), snapshotStore.getRole()); + if (zoneId.equals(entryZoneId)) { + snapshotStoreRefs.add(snapshotStore); + } + } + zoneIds.add(zoneId); + } else { + snapshotStoreRefs = allSnapshotStoreRefs; + for (SnapshotDataStoreVO snapshotStore : snapshotStoreRefs) { + Long entryZoneId = dataStoreMgr.getStoreZoneId(snapshotStore.getDataStoreId(), snapshotStore.getRole()); + if (!zoneIds.contains(entryZoneId)) { + zoneIds.add(entryZoneId); + } + } + } + return new Pair<>(snapshotStoreRefs, zoneIds); + } + @Override @DB @ActionEvent(eventType = EventTypes.EVENT_SNAPSHOT_DELETE, eventDescription = "deleting snapshot", async = true) - public boolean deleteSnapshot(long snapshotId) { + public boolean deleteSnapshot(long snapshotId, Long zoneId) { Account caller = CallContext.current().getCallingAccount(); // Verify parameters - SnapshotVO snapshotCheck = _snapshotDao.findById(snapshotId); + final SnapshotVO snapshotCheck = _snapshotDao.findById(snapshotId); if (snapshotCheck == null) { throw new InvalidParameterValueException("unable to find a snapshot with id " + snapshotId); @@ -608,35 +667,36 @@ public boolean deleteSnapshot(long snapshotId) { _accountMgr.checkAccess(caller, null, true, snapshotCheck); - SnapshotStrategy snapshotStrategy = _storageStrategyFactory.getSnapshotStrategy(snapshotCheck, SnapshotOperation.DELETE); + SnapshotStrategy snapshotStrategy = _storageStrategyFactory.getSnapshotStrategy(snapshotCheck, zoneId, SnapshotOperation.DELETE); if (snapshotStrategy == null) { s_logger.error("Unable to find snapshot strategy to handle snapshot with id '" + snapshotId + "'"); return false; } - - DataStoreRole dataStoreRole = snapshotHelper.getDataStoreRole(snapshotCheck); - - SnapshotDataStoreVO snapshotStoreRef = _snapshotStoreDao.findBySnapshot(snapshotId, dataStoreRole); + Pair, List> storeRefAndZones = getStoreRefsAndZonesForSnapshotDelete(snapshotId, zoneId); + List snapshotStoreRefs = storeRefAndZones.first(); + List zoneIds = storeRefAndZones.second(); try { - boolean result = snapshotStrategy.deleteSnapshot(snapshotId); - + boolean result = snapshotStrategy.deleteSnapshot(snapshotId, zoneId); if (result) { - annotationDao.removeByEntityType(AnnotationService.EntityType.SNAPSHOT.name(), snapshotCheck.getUuid()); - - if (snapshotCheck.getState() == Snapshot.State.BackedUp) { - UsageEventUtils.publishUsageEvent(EventTypes.EVENT_SNAPSHOT_DELETE, snapshotCheck.getAccountId(), snapshotCheck.getDataCenterId(), snapshotId, - snapshotCheck.getName(), null, null, 0L, snapshotCheck.getClass().getName(), snapshotCheck.getUuid()); + for (Long zId : zoneIds) { + if (snapshotCheck.getState() == Snapshot.State.BackedUp) { + UsageEventUtils.publishUsageEvent(EventTypes.EVENT_SNAPSHOT_DELETE, snapshotCheck.getAccountId(), zId, snapshotId, + snapshotCheck.getName(), null, null, 0L, snapshotCheck.getClass().getName(), snapshotCheck.getUuid()); + } } + final SnapshotVO postDeleteSnapshotEntry = _snapshotDao.findById(snapshotId); + if (postDeleteSnapshotEntry == null || Snapshot.State.Destroyed.equals(postDeleteSnapshotEntry.getState())) { + annotationDao.removeByEntityType(AnnotationService.EntityType.SNAPSHOT.name(), snapshotCheck.getUuid()); - if (snapshotCheck.getState() != Snapshot.State.Error && snapshotCheck.getState() != Snapshot.State.Destroyed) { - _resourceLimitMgr.decrementResourceCount(snapshotCheck.getAccountId(), ResourceType.snapshot); + if (snapshotCheck.getState() != Snapshot.State.Error && snapshotCheck.getState() != Snapshot.State.Destroyed) { + _resourceLimitMgr.decrementResourceCount(snapshotCheck.getAccountId(), ResourceType.snapshot); + } } - - if (snapshotCheck.getState() == Snapshot.State.BackedUp) { - if (snapshotStoreRef != null) { + for (SnapshotDataStoreVO snapshotStoreRef : snapshotStoreRefs) { + if (ObjectInDataStoreStateMachine.State.Ready.equals(snapshotStoreRef.getState()) && !DataStoreRole.Primary.equals(snapshotStoreRef.getRole())) { _resourceLimitMgr.decrementResourceCount(snapshotCheck.getAccountId(), ResourceType.secondary_storage, new Long(snapshotStoreRef.getPhysicalSize())); } } @@ -650,18 +710,6 @@ public boolean deleteSnapshot(long snapshotId) { } } - @Override - public String getSecondaryStorageURL(SnapshotVO snapshot) { - SnapshotDataStoreVO snapshotStore = _snapshotStoreDao.findBySnapshot(snapshot.getId(), DataStoreRole.Image); - if (snapshotStore != null) { - DataStore store = dataStoreMgr.getDataStore(snapshotStore.getDataStoreId(), DataStoreRole.Image); - if (store != null) { - return store.getUri(); - } - } - throw new CloudRuntimeException("Can not find secondary storage hosting the snapshot"); - } - @Override public Pair, Integer> listSnapshots(ListSnapshotsCmd cmd) { Long volumeId = cmd.getVolumeId(); @@ -831,12 +879,12 @@ public boolean deleteSnapshotDirsForAccount(long accountId) { s_logger.error("Unable to find snapshot strategy to handle snapshot with id '" + snapshot.getId() + "'"); continue; } - SnapshotDataStoreVO snapshotStoreRef = _snapshotStoreDao.findBySnapshot(snapshot.getId(), DataStoreRole.Image); + List snapshotStoreRefs = _snapshotStoreDao.listReadyBySnapshot(snapshot.getId(), DataStoreRole.Image); - if (snapshotStrategy.deleteSnapshot(snapshot.getId())) { + if (snapshotStrategy.deleteSnapshot(snapshot.getId(), null)) { if (Type.MANUAL == snapshot.getRecurringType()) { _resourceLimitMgr.decrementResourceCount(accountId, ResourceType.snapshot); - if (snapshotStoreRef != null) { + for (SnapshotDataStoreVO snapshotStoreRef : snapshotStoreRefs) { _resourceLimitMgr.decrementResourceCount(accountId, ResourceType.secondary_storage, new Long(snapshotStoreRef.getPhysicalSize())); } } @@ -852,6 +900,23 @@ public boolean deleteSnapshotDirsForAccount(long accountId) { return success; } + protected void validatePolicyZones(List zoneIds, VolumeVO volume, Account caller) { + if (CollectionUtils.isEmpty(zoneIds)) { + return; + } + if (Boolean.FALSE.equals(SnapshotInfo.BackupSnapshotAfterTakingSnapshot.value())) { + throw new InvalidParameterValueException("Backing up of snapshot has been disabled. Snapshot can not be taken for multiple zones"); + } + final DataCenterVO zone = dataCenterDao.findById(volume.getDataCenterId()); + if (DataCenter.Type.Edge.equals(zone.getType())) { + throw new InvalidParameterValueException("Backing up of snapshot is not supported by the zone of the volume. Snapshots can not be taken for multiple zones"); + } + boolean isRootAdminCaller = _accountMgr.isRootAdmin(caller.getId()); + for (Long zoneId : zoneIds) { + getCheckedDestinationZoneForSnapshotCopy(zoneId, isRootAdminCaller); + } + } + @Override @DB @ActionEvent(eventType = EventTypes.EVENT_SNAPSHOT_POLICY_CREATE, eventDescription = "creating snapshot policy") @@ -873,7 +938,8 @@ public SnapshotPolicyVO createPolicy(CreateSnapshotPolicyCmd cmd, Account policy String volumeDescription = volume.getVolumeDescription(); - _accountMgr.checkAccess(CallContext.current().getCallingAccount(), null, true, volume); + final Account caller = CallContext.current().getCallingAccount(); + _accountMgr.checkAccess(caller, null, true, volume); // If display is false we don't actually schedule snapshots. if (volume.getState() != Volume.State.Ready && display) { @@ -952,13 +1018,16 @@ public SnapshotPolicyVO createPolicy(CreateSnapshotPolicyCmd cmd, Account policy } } + final List zoneIds = cmd.getZoneIds(); + validatePolicyZones(zoneIds, volume, caller); + Map tags = cmd.getTags(); boolean active = true; - return persistSnapshotPolicy(volume, schedule, timezoneId, intvType, maxSnaps, display, active, tags); + return persistSnapshotPolicy(volume, schedule, timezoneId, intvType, maxSnaps, display, active, tags, zoneIds); } - protected SnapshotPolicyVO persistSnapshotPolicy(VolumeVO volume, String schedule, String timezone, IntervalType intervalType, int maxSnaps, boolean display, boolean active, Map tags) { + protected SnapshotPolicyVO persistSnapshotPolicy(VolumeVO volume, String schedule, String timezone, IntervalType intervalType, int maxSnaps, boolean display, boolean active, Map tags, List zoneIds) { long volumeId = volume.getId(); String volumeDescription = volume.getVolumeDescription(); @@ -966,7 +1035,7 @@ protected SnapshotPolicyVO persistSnapshotPolicy(VolumeVO volume, String schedul boolean isLockAcquired = createSnapshotPolicyLock.lock(5); if (!isLockAcquired) { - throw new CloudRuntimeException(String.format("Unable to aquire lock for creating snapshot policy [%s] for %s.", intervalType, volumeDescription)); + throw new CloudRuntimeException(String.format("Unable to acquire lock for creating snapshot policy [%s] for %s.", intervalType, volumeDescription)); } s_logger.debug(String.format("Acquired lock for creating snapshot policy [%s] for volume %s.", intervalType, volumeDescription)); @@ -975,9 +1044,9 @@ protected SnapshotPolicyVO persistSnapshotPolicy(VolumeVO volume, String schedul SnapshotPolicyVO policy = _snapshotPolicyDao.findOneByVolumeInterval(volumeId, intervalType); if (policy == null) { - policy = createSnapshotPolicy(volumeId, schedule, timezone, intervalType, maxSnaps, display); + policy = createSnapshotPolicy(volumeId, schedule, timezone, intervalType, maxSnaps, display, zoneIds); } else { - updateSnapshotPolicy(policy, schedule, timezone, intervalType, maxSnaps, active, display); + updateSnapshotPolicy(policy, schedule, timezone, intervalType, maxSnaps, active, display, zoneIds); } createTagsForSnapshotPolicy(tags, policy); @@ -989,15 +1058,22 @@ protected SnapshotPolicyVO persistSnapshotPolicy(VolumeVO volume, String schedul } } - protected SnapshotPolicyVO createSnapshotPolicy(long volumeId, String schedule, String timezone, IntervalType intervalType, int maxSnaps, boolean display) { + protected SnapshotPolicyVO createSnapshotPolicy(long volumeId, String schedule, String timezone, IntervalType intervalType, int maxSnaps, boolean display, List zoneIds) { SnapshotPolicyVO policy = new SnapshotPolicyVO(volumeId, schedule, timezone, intervalType, maxSnaps, display); policy = _snapshotPolicyDao.persist(policy); + if (CollectionUtils.isNotEmpty(zoneIds)) { + List details = new ArrayList<>(); + for (Long zoneId : zoneIds) { + details.add(new SnapshotPolicyDetailVO(policy.getId(), ApiConstants.ZONE_ID, String.valueOf(zoneId))); + } + snapshotPolicyDetailsDao.saveDetails(details); + } _snapSchedMgr.scheduleNextSnapshotJob(policy); s_logger.debug(String.format("Created snapshot policy %s.", new ReflectionToStringBuilder(policy, ToStringStyle.JSON_STYLE).setExcludeFieldNames("id", "uuid", "active"))); return policy; } - protected void updateSnapshotPolicy(SnapshotPolicyVO policy, String schedule, String timezone, IntervalType intervalType, int maxSnaps, boolean active, boolean display) { + protected void updateSnapshotPolicy(SnapshotPolicyVO policy, String schedule, String timezone, IntervalType intervalType, int maxSnaps, boolean active, boolean display, List zoneIds) { String previousPolicy = new ReflectionToStringBuilder(policy, ToStringStyle.JSON_STYLE).setExcludeFieldNames("id", "uuid").toString(); boolean previousDisplay = policy.isDisplay(); policy.setSchedule(schedule); @@ -1007,6 +1083,15 @@ protected void updateSnapshotPolicy(SnapshotPolicyVO policy, String schedule, St policy.setActive(active); policy.setDisplay(display); _snapshotPolicyDao.update(policy.getId(), policy); + if (CollectionUtils.isNotEmpty(zoneIds)) { + List details = snapshotPolicyDetailsDao.listDetails(policy.getId()); + details = details.stream().filter(d -> !ApiConstants.ZONE_ID.equals(d.getName())).collect(Collectors.toList()); + for (Long zoneId : zoneIds) { + details.add(new SnapshotPolicyDetailVO(policy.getId(), ApiConstants.ZONE_ID, String.valueOf(zoneId))); + } + snapshotPolicyDetailsDao.saveDetails(details); + } + _snapSchedMgr.scheduleOrCancelNextSnapshotJobOnDisplayChange(policy, previousDisplay); taggedResourceService.deleteTags(Collections.singletonList(policy.getUuid()), ResourceObjectType.SnapshotPolicy, null); s_logger.debug(String.format("Updated snapshot policy %s to %s.", previousPolicy, new ReflectionToStringBuilder(policy, ToStringStyle.JSON_STYLE) @@ -1027,10 +1112,12 @@ public void copySnapshotPoliciesBetweenVolumes(VolumeVO srcVolume, VolumeVO dest s_logger.debug(String.format("Copying snapshot policies %s from volume %s to volume %s.", ReflectionToStringBuilderUtils.reflectOnlySelectedFields(policies, "id", "uuid"), srcVolume.getVolumeDescription(), destVolume.getVolumeDescription())); - policies.forEach(policy -> - persistSnapshotPolicy(destVolume, policy.getSchedule(), policy.getTimezone(), intervalTypes[policy.getInterval()], policy.getMaxSnaps(), - policy.isDisplay(), policy.isActive(), taggedResourceService.getTagsFromResource(ResourceObjectType.SnapshotPolicy, policy.getId())) - ); + for (SnapshotPolicyVO policy : policies) { + List details = snapshotPolicyDetailsDao.findDetails(policy.getId(), ApiConstants.ZONE_ID); + List zoneIds = details.stream().map(d -> Long.valueOf(d.getValue())).collect(Collectors.toList()); + persistSnapshotPolicy(destVolume, policy.getSchedule(), policy.getTimezone(), intervalTypes[policy.getInterval()], policy.getMaxSnaps(), + policy.isDisplay(), policy.isActive(), taggedResourceService.getTagsFromResource(ResourceObjectType.SnapshotPolicy, policy.getId()), zoneIds); + } } protected boolean deletePolicy(Long policyId) { @@ -1266,7 +1353,7 @@ public SnapshotInfo takeSnapshot(VolumeInfo volume) throws ResourceAllocationExc boolean backupSnapToSecondary = isBackupSnapshotToSecondaryForZone(snapshot.getDataCenterId()); if (backupSnapToSecondary) { - backupSnapshotToSecondary(payload.getAsyncBackup(), snapshotStrategy, snapshotOnPrimary); + backupSnapshotToSecondary(payload.getAsyncBackup(), snapshotStrategy, snapshotOnPrimary, payload.getZoneIds()); } else { s_logger.debug("skipping backup of snapshot [uuid=" + snapshot.getUuid() + "] to secondary due to configuration"); snapshotOnPrimary.markBackedUp(); @@ -1274,18 +1361,24 @@ public SnapshotInfo takeSnapshot(VolumeInfo volume) throws ResourceAllocationExc try { postCreateSnapshot(volume.getId(), snapshotId, payload.getSnapshotPolicyId()); + snapshotZoneDao.addSnapshotToZone(snapshotId, snapshot.getDataCenterId()); DataStoreRole dataStoreRole = backupSnapToSecondary ? snapshotHelper.getDataStoreRole(snapshot) : DataStoreRole.Primary; - SnapshotDataStoreVO snapshotStoreRef = _snapshotStoreDao.findBySnapshot(snapshotId, dataStoreRole); - if (snapshotStoreRef == null) { + List snapshotStoreRefs = _snapshotStoreDao.listReadyBySnapshot(snapshotId, dataStoreRole); + if (CollectionUtils.isEmpty(snapshotStoreRefs)) { throw new CloudRuntimeException(String.format("Could not find snapshot %s [%s] on [%s]", snapshot.getName(), snapshot.getUuid(), snapshot.getLocationType())); } + SnapshotDataStoreVO snapshotStoreRef = snapshotStoreRefs.get(0); UsageEventUtils.publishUsageEvent(EventTypes.EVENT_SNAPSHOT_CREATE, snapshot.getAccountId(), snapshot.getDataCenterId(), snapshotId, snapshot.getName(), null, null, snapshotStoreRef.getPhysicalSize(), volume.getSize(), snapshot.getClass().getName(), snapshot.getUuid()); // Correct the resource count of snapshot in case of delta snapshots. _resourceLimitMgr.decrementResourceCount(snapshotOwner.getId(), ResourceType.secondary_storage, new Long(volume.getSize() - snapshotStoreRef.getPhysicalSize())); + + if (!payload.getAsyncBackup() && backupSnapToSecondary) { + copyNewSnapshotToZones(snapshotId, snapshot.getDataCenterId(), payload.getZoneIds()); + } } catch (Exception e) { s_logger.debug("post process snapshot failed", e); } @@ -1307,9 +1400,9 @@ public SnapshotInfo takeSnapshot(VolumeInfo volume) throws ResourceAllocationExc return snapshot; } - protected void backupSnapshotToSecondary(boolean asyncBackup, SnapshotStrategy snapshotStrategy, SnapshotInfo snapshotOnPrimary) { + protected void backupSnapshotToSecondary(boolean asyncBackup, SnapshotStrategy snapshotStrategy, SnapshotInfo snapshotOnPrimary, List zoneIds) { if (asyncBackup) { - backupSnapshotExecutor.schedule(new BackupSnapshotTask(snapshotOnPrimary, snapshotBackupRetries - 1, snapshotStrategy), 0, TimeUnit.SECONDS); + backupSnapshotExecutor.schedule(new BackupSnapshotTask(snapshotOnPrimary, snapshotBackupRetries - 1, snapshotStrategy, zoneIds), 0, TimeUnit.SECONDS); } else { SnapshotInfo backupedSnapshot = snapshotStrategy.backupSnapshot(snapshotOnPrimary); if (backupedSnapshot != null) { @@ -1323,10 +1416,13 @@ protected class BackupSnapshotTask extends ManagedContextRunnable { int attempts; SnapshotStrategy snapshotStrategy; - public BackupSnapshotTask(SnapshotInfo snap, int maxRetries, SnapshotStrategy strategy) { + List zoneIds; + + public BackupSnapshotTask(SnapshotInfo snap, int maxRetries, SnapshotStrategy strategy, List zoneIds) { snapshot = snap; attempts = maxRetries; snapshotStrategy = strategy; + this.zoneIds = zoneIds; } @Override @@ -1338,11 +1434,12 @@ protected void runInContext() { if (backupedSnapshot != null) { snapshotStrategy.postSnapshotCreation(snapshot); + copyNewSnapshotToZones(snapshot.getId(), snapshot.getDataCenterId(), zoneIds); } } catch (final Exception e) { if (attempts >= 0) { s_logger.debug("Backing up of snapshot failed, for snapshot with ID " + snapshot.getSnapshotId() + ", left with " + attempts + " more attempts"); - backupSnapshotExecutor.schedule(new BackupSnapshotTask(snapshot, --attempts, snapshotStrategy), snapshotBackupRetryInterval, TimeUnit.SECONDS); + backupSnapshotExecutor.schedule(new BackupSnapshotTask(snapshot, --attempts, snapshotStrategy, zoneIds), snapshotBackupRetryInterval, TimeUnit.SECONDS); } else { s_logger.debug("Done with " + snapshotBackupRetries + " attempts in backing up of snapshot with ID " + snapshot.getSnapshotId()); snapshotSrv.cleanupOnSnapshotBackupFailure(snapshot); @@ -1391,7 +1488,7 @@ public boolean start() { List snapshots = _snapshotDao.listAllByStatus(Snapshot.State.Destroying); for (SnapshotVO snapshotVO : snapshots) { try { - if (!deleteSnapshot(snapshotVO.getId())) { + if (!deleteSnapshot(snapshotVO.getId(), null)) { s_logger.debug("Failed to delete snapshot in destroying state with id " + snapshotVO.getUuid()); } } catch (Exception e) { @@ -1471,7 +1568,7 @@ public boolean backedUpSnapshotsExistsForVolume(Volume volume) { @Override public void cleanupSnapshotsByVolume(Long volumeId) { - List infos = snapshotFactory.getSnapshots(volumeId, DataStoreRole.Primary); + List infos = snapshotFactory.getSnapshotsForVolumeAndStoreRole(volumeId, DataStoreRole.Primary); for (SnapshotInfo info : infos) { try { if (info != null) { @@ -1486,11 +1583,11 @@ public void cleanupSnapshotsByVolume(Long volumeId) { @Override public Snapshot allocSnapshot(Long volumeId, Long policyId, String snapshotName, Snapshot.LocationType locationType) throws ResourceAllocationException { - return allocSnapshot(volumeId, policyId, snapshotName, locationType, false); + return allocSnapshot(volumeId, policyId, snapshotName, locationType, false, null); } @Override - public Snapshot allocSnapshot(Long volumeId, Long policyId, String snapshotName, Snapshot.LocationType locationType, Boolean isFromVmSnapshot) throws ResourceAllocationException { + public Snapshot allocSnapshot(Long volumeId, Long policyId, String snapshotName, Snapshot.LocationType locationType, Boolean isFromVmSnapshot, List zoneIds) throws ResourceAllocationException { Account caller = CallContext.current().getCallingAccount(); VolumeInfo volume = volFactory.getVolume(volumeId); supportedByHypervisor(volume, isFromVmSnapshot); @@ -1568,4 +1665,268 @@ public void markVolumeSnapshotsAsDestroyed(Volume volume) { } } } + + private boolean checkAndProcessSnapshotAlreadyExistInStore(long snapshotId, DataStore dstSecStore) { + SnapshotDataStoreVO dstSnapshotStore = _snapshotStoreDao.findByStoreSnapshot(DataStoreRole.Image, dstSecStore.getId(), snapshotId); + if (dstSnapshotStore == null) { + return false; + } + if (dstSnapshotStore.getState() == ObjectInDataStoreStateMachine.State.Ready) { + if (!dstSnapshotStore.isDisplay()) { + s_logger.debug(String.format("Snapshot ID: %d is in ready state on image store ID: %d, marking it displayable for view", snapshotId, dstSnapshotStore.getDataStoreId())); + dstSnapshotStore.setDisplay(true); + _snapshotStoreDao.update(dstSnapshotStore.getId(), dstSnapshotStore); + } + return true; // already downloaded on this image store + } + if (List.of(VMTemplateStorageResourceAssoc.Status.ABANDONED, + VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR, + VMTemplateStorageResourceAssoc.Status.NOT_DOWNLOADED, + VMTemplateStorageResourceAssoc.Status.UNKNOWN).contains(dstSnapshotStore.getDownloadState()) || + !List.of(ObjectInDataStoreStateMachine.State.Creating, + ObjectInDataStoreStateMachine.State.Copying).contains(dstSnapshotStore.getState())) { + _snapshotStoreDao.removeBySnapshotStore(snapshotId, dstSecStore.getId(), DataStoreRole.Image); + } + return false; + } + + @DB + private boolean copySnapshotToZone(SnapshotDataStoreVO snapshotDataStoreVO, DataStore srcSecStore, + DataCenterVO dstZone, DataStore dstSecStore, Account account) + throws ResourceAllocationException { + final long snapshotId = snapshotDataStoreVO.getSnapshotId(); + final long dstZoneId = dstZone.getId(); + if (checkAndProcessSnapshotAlreadyExistInStore(snapshotId, dstSecStore)) { + return true; + } + _resourceLimitMgr.checkResourceLimit(account, ResourceType.secondary_storage, snapshotDataStoreVO.getSize()); + // snapshotId may refer to ID of a removed parent snapshot + SnapshotInfo snapshotOnSecondary = snapshotFactory.getSnapshot(snapshotId, srcSecStore); + String copyUrl = null; + try { + AsyncCallFuture future = snapshotSrv.queryCopySnapshot(snapshotOnSecondary); + CreateCmdResult result = future.get(); + if (!result.isFailed()) { + copyUrl = result.getPath(); + } + } catch (InterruptedException | ExecutionException | ResourceUnavailableException ex) { + s_logger.error(String.format("Failed to prepare URL for copy for snapshot ID: %d on store: %s", snapshotId, srcSecStore.getName()), ex); + } + if (StringUtils.isEmpty(copyUrl)) { + s_logger.error(String.format("Unable to prepare URL for copy for snapshot ID: %d on store: %s", snapshotId, srcSecStore.getName())); + return false; + } + s_logger.debug(String.format("Copying snapshot ID: %d to destination zones using download URL: %s", snapshotId, copyUrl)); + try { + AsyncCallFuture future = snapshotSrv.copySnapshot(snapshotOnSecondary, copyUrl, dstSecStore); + SnapshotResult result = future.get(); + if (result.isFailed()) { + s_logger.debug(String.format("Copy snapshot ID: %d failed for image store %s: %s", snapshotId, dstSecStore.getName(), result.getResult())); + return false; + } + snapshotZoneDao.addSnapshotToZone(snapshotId, dstZoneId); + _resourceLimitMgr.incrementResourceCount(account.getId(), ResourceType.secondary_storage, snapshotDataStoreVO.getSize()); + if (account.getId() != Account.ACCOUNT_ID_SYSTEM) { + SnapshotVO snapshotVO = _snapshotDao.findByIdIncludingRemoved(snapshotId); + UsageEventUtils.publishUsageEvent(EventTypes.EVENT_SNAPSHOT_COPY, account.getId(), dstZoneId, snapshotId, null, null, null, snapshotVO.getSize(), + snapshotVO.getSize(), snapshotVO.getClass().getName(), snapshotVO.getUuid()); + } + return true; + } catch (InterruptedException | ExecutionException | ResourceUnavailableException ex) { + s_logger.debug(String.format("Failed to copy snapshot ID: %d to image store: %s", snapshotId, dstSecStore.getName())); + } + return false; + } + + @DB + private boolean copySnapshotChainToZone(SnapshotVO snapshotVO, DataStore srcSecStore, DataCenterVO destZone, Account account) + throws StorageUnavailableException, ResourceAllocationException { + final long snapshotId = snapshotVO.getId(); + final long destZoneId = destZone.getId(); + SnapshotDataStoreVO currentSnap = _snapshotStoreDao.findByStoreSnapshot(DataStoreRole.Image, srcSecStore.getId(), snapshotId);; + List snapshotChain = new ArrayList<>(); + long size = 0L; + DataStore dstSecStore = null; + do { + dstSecStore = getSnapshotZoneImageStore(currentSnap.getSnapshotId(), destZone.getId()); + if (dstSecStore != null) { + s_logger.debug(String.format("Snapshot ID: %d is already present in secondary storage: %s" + + " in zone %s in ready state, don't need to copy any further", + currentSnap.getSnapshotId(), dstSecStore.getName(), destZone)); + if (snapshotId == currentSnap.getSnapshotId()) { + checkAndProcessSnapshotAlreadyExistInStore(snapshotId, dstSecStore); + } + break; + } + snapshotChain.add(currentSnap); + size += currentSnap.getSize(); + currentSnap = currentSnap.getParentSnapshotId() == 0 ? + null : + _snapshotStoreDao.findByStoreSnapshot(DataStoreRole.Image, srcSecStore.getId(), currentSnap.getParentSnapshotId()); + } while (currentSnap != null); + if (CollectionUtils.isEmpty(snapshotChain)) { + return true; + } + try { + _resourceLimitMgr.checkResourceLimit(account, ResourceType.secondary_storage, size); + } catch (ResourceAllocationException e) { + s_logger.error(String.format("Unable to allocate secondary storage resources for snapshot chain for %s with size: %d", snapshotVO, size), e); + return false; + } + Collections.reverse(snapshotChain); + if (dstSecStore == null) { + // find all eligible image stores for the destination zone + List dstSecStores = dataStoreMgr.getImageStoresByScopeExcludingReadOnly(new ZoneScope(destZoneId)); + if (CollectionUtils.isEmpty(dstSecStores)) { + throw new StorageUnavailableException("Destination zone is not ready, no image store associated", DataCenter.class, destZoneId); + } + dstSecStore = dataStoreMgr.getImageStoreWithFreeCapacity(dstSecStores); + if (dstSecStore == null) { + throw new StorageUnavailableException("Destination zone is not ready, no image store with free capacity", DataCenter.class, destZoneId); + } + } + s_logger.debug(String.format("Copying snapshot chain for snapshot ID: %d on secondary store: %s of zone ID: %d", snapshotId, dstSecStore.getName(), destZoneId)); + for (SnapshotDataStoreVO snapshotDataStoreVO : snapshotChain) { + if (!copySnapshotToZone(snapshotDataStoreVO, srcSecStore, destZone, dstSecStore, account)) { + s_logger.error(String.format("Failed to copy snapshot: %s to zone: %s due to failure to copy snapshot ID: %d from snapshot chain", + snapshotVO, destZone, snapshotDataStoreVO.getSnapshotId())); + return false; + } + } + return true; + } + + @DB + private List copySnapshotToZones(SnapshotVO snapshotVO, DataStore srcSecStore, List dstZones) throws StorageUnavailableException, ResourceAllocationException { + AccountVO account = _accountDao.findById(snapshotVO.getAccountId()); + List failedZones = new ArrayList<>(); + for (DataCenterVO destZone : dstZones) { + if (!copySnapshotChainToZone(snapshotVO, srcSecStore, destZone, account)) { + failedZones.add(destZone.getName()); + } + } + return failedZones; + } + + protected Pair getCheckedSnapshotForCopy(final long snapshotId, final List destZoneIds, Long sourceZoneId) { + SnapshotVO snapshot = _snapshotDao.findById(snapshotId); + if (snapshot == null) { + throw new InvalidParameterValueException("Unable to find snapshot with id"); + } + // Verify snapshot is BackedUp and is on secondary store + if (!Snapshot.State.BackedUp.equals(snapshot.getState())) { + throw new InvalidParameterValueException("Snapshot is not backed up"); + } + if (snapshot.getLocationType() != null && !Snapshot.LocationType.SECONDARY.equals(snapshot.getLocationType())) { + throw new InvalidParameterValueException("Snapshot is not backed up"); + } + if (CollectionUtils.isEmpty(destZoneIds)) { + throw new InvalidParameterValueException("Please specify valid destination zone(s)."); + } + Volume volume = _volsDao.findById(snapshot.getVolumeId()); + if (sourceZoneId == null) { + sourceZoneId = volume.getDataCenterId(); + } + if (destZoneIds.contains(sourceZoneId)) { + throw new InvalidParameterValueException("Please specify different source and destination zones."); + } + DataCenterVO sourceZone = dataCenterDao.findById(sourceZoneId); + if (sourceZone == null) { + throw new InvalidParameterValueException("Please specify a valid source zone."); + } + return new Pair<>(snapshot, sourceZoneId); + } + + protected DataCenterVO getCheckedDestinationZoneForSnapshotCopy(long zoneId, boolean isRootAdmin) { + DataCenterVO dstZone = dataCenterDao.findById(zoneId); + if (dstZone == null) { + throw new InvalidParameterValueException("Please specify a valid destination zone."); + } + if (Grouping.AllocationState.Disabled.equals(dstZone.getAllocationState()) && !isRootAdmin) { + throw new PermissionDeniedException("Cannot perform this operation, Zone is currently disabled: " + dstZone.getName()); + } + if (DataCenter.Type.Edge.equals(dstZone.getType())) { + s_logger.error(String.format("Edge zone %s specified for snapshot copy", dstZone)); + throw new InvalidParameterValueException(String.format("Snapshot copy is not supported by zone %s", dstZone.getName())); + } + return dstZone; + } + + @Override + @ActionEvent(eventType = EventTypes.EVENT_SNAPSHOT_COPY, eventDescription = "copying snapshot", create = false) + public Snapshot copySnapshot(CopySnapshotCmd cmd) throws StorageUnavailableException, ResourceAllocationException { + final Long snapshotId = cmd.getId(); + Long sourceZoneId = cmd.getSourceZoneId(); + List destZoneIds = cmd.getDestinationZoneIds(); + Account caller = CallContext.current().getCallingAccount(); + Pair snapshotZonePair = getCheckedSnapshotForCopy(snapshotId, destZoneIds, sourceZoneId); + SnapshotVO snapshot = snapshotZonePair.first(); + sourceZoneId = snapshotZonePair.second(); + Map dataCenterVOs = new HashMap<>(); + boolean isRootAdminCaller = _accountMgr.isRootAdmin(caller.getId()); + for (Long destZoneId: destZoneIds) { + DataCenterVO dstZone = getCheckedDestinationZoneForSnapshotCopy(destZoneId, isRootAdminCaller); + dataCenterVOs.put(destZoneId, dstZone); + } + _accountMgr.checkAccess(caller, SecurityChecker.AccessType.OperateEntry, true, snapshot); + DataStore srcSecStore = getSnapshotZoneImageStore(snapshotId, sourceZoneId); + if (srcSecStore == null) { + throw new InvalidParameterValueException(String.format("There is no snapshot ID: %s ready on image store", snapshot.getUuid())); + } + List failedZones = copySnapshotToZones(snapshot, srcSecStore, new ArrayList<>(dataCenterVOs.values())); + if (destZoneIds.size() > failedZones.size()){ + if (!failedZones.isEmpty()) { + s_logger.error(String.format("There were failures when copying snapshot to zones: %s", + StringUtils.joinWith(", ", failedZones.toArray()))); + } + return snapshot; + } else { + throw new CloudRuntimeException("Failed to copy snapshot"); + } + } + + protected void copyNewSnapshotToZones(long snapshotId, long zoneId, List destZoneIds) { + if (CollectionUtils.isEmpty(destZoneIds)) { + return; + } + List failedZones = new ArrayList<>(); + SnapshotVO snapshotVO = _snapshotDao.findById(snapshotId); + long startEventId = ActionEventUtils.onStartedActionEvent(CallContext.current().getCallingUserId(), + CallContext.current().getCallingAccountId(), EventTypes.EVENT_SNAPSHOT_COPY, + String.format("Copying snapshot ID: %s", snapshotVO.getUuid()), snapshotId, + ApiCommandResourceType.Snapshot.toString(), true, 0); + DataStore dataStore = getSnapshotZoneImageStore(snapshotId, zoneId); + String completedEventLevel = EventVO.LEVEL_ERROR; + String completedEventMsg = String.format("Copying snapshot ID: %s failed", snapshotVO.getUuid()); + if (dataStore == null) { + s_logger.error(String.format("Unable to find an image store for zone ID: %d where snapshot %s is in Ready state", zoneId, snapshotVO)); + ActionEventUtils.onCompletedActionEvent(CallContext.current().getCallingUserId(), + CallContext.current().getCallingAccountId(), completedEventLevel, EventTypes.EVENT_SNAPSHOT_COPY, + completedEventMsg, snapshotId, ApiCommandResourceType.Snapshot.toString(), startEventId); + return; + } + List dataCenterVOs = new ArrayList<>(); + for (Long destZoneId: destZoneIds) { + DataCenterVO dstZone = dataCenterDao.findById(destZoneId); + dataCenterVOs.add(dstZone); + } + try { + failedZones = copySnapshotToZones(snapshotVO, dataStore, dataCenterVOs); + if (CollectionUtils.isNotEmpty(failedZones)) { + s_logger.error(String.format("There were failures while copying snapshot %s to zones: %s", + snapshotVO, StringUtils.joinWith(", ", failedZones.toArray()))); + } + } catch (ResourceAllocationException | StorageUnavailableException | CloudRuntimeException e) { + s_logger.error(String.format("Error while copying snapshot %s to zones: %s", snapshotVO, StringUtils.joinWith(",", destZoneIds.toArray()))); + } + if (failedZones.size() < destZoneIds.size()) { + final List failedZonesFinal = failedZones; + String zoneNames = StringUtils.joinWith(", ", dataCenterVOs.stream().filter(x -> !failedZonesFinal.contains(x.getUuid())).map(DataCenterVO::getName).collect(Collectors.toList())); + completedEventLevel = EventVO.LEVEL_INFO; + completedEventMsg = String.format("Completed copying snapshot ID: %s to zone(s): %s", snapshotVO.getUuid(), zoneNames); + } + ActionEventUtils.onCompletedActionEvent(CallContext.current().getCallingUserId(), + CallContext.current().getCallingAccountId(), completedEventLevel, EventTypes.EVENT_SNAPSHOT_COPY, + completedEventMsg, snapshotId, ApiCommandResourceType.Snapshot.toString(), startEventId); + } } diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java index bb8affc18700..43a3de2078d1 100755 --- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java @@ -34,8 +34,6 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; -import com.cloud.user.UserData; -import com.cloud.storage.VolumeApiService; import org.apache.cloudstack.acl.SecurityChecker.AccessType; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseListTemplateOrIsoPermissionsCmd; @@ -85,6 +83,7 @@ import org.apache.cloudstack.framework.messagebus.MessageBus; import org.apache.cloudstack.framework.messagebus.PublishScope; import org.apache.cloudstack.managed.context.ManagedContextRunnable; +import org.apache.cloudstack.snapshot.SnapshotHelper; import org.apache.cloudstack.storage.command.AttachCommand; import org.apache.cloudstack.storage.command.CommandResult; import org.apache.cloudstack.storage.command.DettachCommand; @@ -164,6 +163,7 @@ import com.cloud.storage.VMTemplateVO; import com.cloud.storage.VMTemplateZoneVO; import com.cloud.storage.Volume; +import com.cloud.storage.VolumeApiService; import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.GuestOSDao; import com.cloud.storage.dao.LaunchPermissionDao; @@ -181,6 +181,7 @@ import com.cloud.user.AccountService; import com.cloud.user.AccountVO; import com.cloud.user.ResourceLimitService; +import com.cloud.user.UserData; import com.cloud.user.dao.AccountDao; import com.cloud.uservm.UserVm; import com.cloud.utils.DateUtil; @@ -209,7 +210,6 @@ import com.google.common.base.Joiner; import com.google.gson.Gson; import com.google.gson.GsonBuilder; -import org.apache.cloudstack.snapshot.SnapshotHelper; public class TemplateManagerImpl extends ManagerBase implements TemplateManager, TemplateApiService, Configurable { private final static Logger s_logger = Logger.getLogger(TemplateManagerImpl.class); @@ -1630,7 +1630,12 @@ public VirtualMachineTemplate createPrivateTemplate(CreateTemplateCmd command) t long zoneId = 0; if (snapshotId != null) { snapshot = _snapshotDao.findById(snapshotId); - zoneId = snapshot.getDataCenterId(); + if (command.getZoneId() == null) { + VolumeVO snapshotVolume = _volumeDao.findByIdIncludingRemoved(snapshot.getVolumeId()); + zoneId = snapshotVolume.getDataCenterId(); + } else { + zoneId = command.getZoneId(); + } } else if (volumeId != null) { volume = _volumeDao.findById(volumeId); zoneId = volume.getDataCenterId(); @@ -1645,7 +1650,7 @@ public VirtualMachineTemplate createPrivateTemplate(CreateTemplateCmd command) t DataStoreRole dataStoreRole = snapshotHelper.getDataStoreRole(snapshot); kvmSnapshotOnlyInPrimaryStorage = snapshotHelper.isKvmSnapshotOnlyInPrimaryStorage(snapshot, dataStoreRole); - snapInfo = _snapshotFactory.getSnapshot(snapshotId, dataStoreRole); + snapInfo = _snapshotFactory.getSnapshotWithRoleAndZone(snapshotId, dataStoreRole, zoneId); if (dataStoreRole == DataStoreRole.Image || kvmSnapshotOnlyInPrimaryStorage) { snapInfo = snapshotHelper.backupSnapshotToSecondaryStorageIfNotExists(snapInfo, dataStoreRole, snapshot, kvmSnapshotOnlyInPrimaryStorage); _accountMgr.checkAccess(caller, null, true, snapInfo); @@ -1781,12 +1786,16 @@ public VMTemplateVO createPrivateTemplateRecord(CreateTemplateCmd cmd, Account t boolean isDynamicScalingEnabled = cmd.isDynamicallyScalable(); // check whether template owner can create public templates boolean allowPublicUserTemplates = AllowPublicUserTemplates.valueIn(templateOwner.getId()); + final Long zoneId = cmd.getZoneId(); if (!isAdmin && !allowPublicUserTemplates && isPublic) { throw new PermissionDeniedException("Failed to create template " + name + ", only private templates can be created."); } Long volumeId = cmd.getVolumeId(); Long snapshotId = cmd.getSnapshotId(); + if (zoneId != null && snapshotId == null) { + throw new InvalidParameterValueException("Failed to create private template record, zone ID can only be specified together with snapshot ID."); + } if ((volumeId == null) && (snapshotId == null)) { throw new InvalidParameterValueException("Failed to create private template record, neither volume ID nor snapshot ID were specified."); } @@ -1860,6 +1869,16 @@ public VMTemplateVO createPrivateTemplateRecord(CreateTemplateCmd cmd, Account t hyperType = snapshot.getHypervisorType(); } + if (zoneId != null) { + DataCenterVO zone = _dcDao.findById(zoneId); + if (zone == null) { + throw new InvalidParameterValueException("Failed to create private template record, invalid zone specified"); + } + if (DataCenter.Type.Edge.equals(zone.getType())) { + throw new InvalidParameterValueException("Failed to create private template record, Edge zones do not support template creation from snapshots"); + } + } + _resourceLimitMgr.checkResourceLimit(templateOwner, ResourceType.template); _resourceLimitMgr.checkResourceLimit(templateOwner, ResourceType.secondary_storage, new Long(volume != null ? volume.getSize() : snapshot.getSize()).longValue()); diff --git a/server/src/main/java/org/apache/cloudstack/snapshot/SnapshotHelper.java b/server/src/main/java/org/apache/cloudstack/snapshot/SnapshotHelper.java index b00612dca658..a11593a86080 100644 --- a/server/src/main/java/org/apache/cloudstack/snapshot/SnapshotHelper.java +++ b/server/src/main/java/org/apache/cloudstack/snapshot/SnapshotHelper.java @@ -19,17 +19,6 @@ package org.apache.cloudstack.snapshot; -import com.cloud.hypervisor.Hypervisor; -import com.cloud.hypervisor.Hypervisor.HypervisorType; -import com.cloud.storage.DataStoreRole; -import com.cloud.storage.Snapshot; -import com.cloud.storage.SnapshotVO; -import com.cloud.storage.VolumeVO; -import com.cloud.storage.Storage.StoragePoolType; -import com.cloud.storage.dao.SnapshotDao; - -import com.cloud.utils.exception.CloudRuntimeException; - import java.util.Arrays; import java.util.HashSet; import java.util.List; @@ -38,6 +27,7 @@ import java.util.stream.Collectors; import javax.inject.Inject; + import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreCapabilities; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; @@ -57,6 +47,16 @@ import org.apache.commons.lang3.builder.ToStringStyle; import org.apache.log4j.Logger; +import com.cloud.hypervisor.Hypervisor; +import com.cloud.hypervisor.Hypervisor.HypervisorType; +import com.cloud.storage.DataStoreRole; +import com.cloud.storage.Snapshot; +import com.cloud.storage.SnapshotVO; +import com.cloud.storage.Storage.StoragePoolType; +import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.SnapshotDao; +import com.cloud.utils.exception.CloudRuntimeException; + public class SnapshotHelper { private final Logger logger = Logger.getLogger(this.getClass()); @@ -110,8 +110,13 @@ public void expungeTemporarySnapshot(boolean kvmSnapshotOnlyInPrimaryStorage, Sn logger.warn(String.format("Unable to delete the temporary snapshot [%s] on secondary storage due to [%s]. We still will expunge the database reference, consider" + " manually deleting the file [%s].", snapInfo.getId(), ex.getMessage(), snapInfo.getPath()), ex); } - - snapshotDataStoreDao.expungeReferenceBySnapshotIdAndDataStoreRole(snapInfo.getId(), DataStoreRole.Image); + long storeId = snapInfo.getDataStore().getId(); + if (!DataStoreRole.Image.equals(snapInfo.getDataStore().getRole())) { + long zoneId = dataStorageManager.getStoreZoneId(storeId, snapInfo.getDataStore().getRole()); + SnapshotInfo imageStoreSnapInfo = snapshotFactory.getSnapshotWithRoleAndZone(snapInfo.getId(), DataStoreRole.Image, zoneId); + storeId = imageStoreSnapInfo.getDataStore().getId(); + } + snapshotDataStoreDao.expungeReferenceBySnapshotIdAndDataStoreRole(snapInfo.getId(), storeId, DataStoreRole.Image); } /** @@ -127,12 +132,12 @@ public SnapshotInfo backupSnapshotToSecondaryStorageIfNotExists(SnapshotInfo sna return snapInfo; } - snapInfo = getSnapshotInfoByIdAndRole(snapshot.getId(), DataStoreRole.Primary); + snapInfo = getSnapshotInfoByIdAndRole(snapshot.getId(), DataStoreRole.Primary, null); SnapshotStrategy snapshotStrategy = storageStrategyFactory.getSnapshotStrategy(snapshot, SnapshotStrategy.SnapshotOperation.BACKUP); snapshotStrategy.backupSnapshot(snapInfo); - return getSnapshotInfoByIdAndRole(snapshot.getId(), kvmSnapshotOnlyInPrimaryStorage ? DataStoreRole.Image : dataStoreRole); + return getSnapshotInfoByIdAndRole(snapshot.getId(), kvmSnapshotOnlyInPrimaryStorage ? DataStoreRole.Image : dataStoreRole, dataStorageManager.getStoreZoneId(snapInfo.getDataStore().getId(), snapInfo.getDataStore().getRole())); } /** @@ -140,8 +145,13 @@ public SnapshotInfo backupSnapshotToSecondaryStorageIfNotExists(SnapshotInfo sna * @return The snapshot info if it exists, else throws an exception. * @throws CloudRuntimeException */ - protected SnapshotInfo getSnapshotInfoByIdAndRole(long snapshotId, DataStoreRole dataStoreRole) throws CloudRuntimeException{ - SnapshotInfo snapInfo = snapshotFactory.getSnapshot(snapshotId, dataStoreRole); + protected SnapshotInfo getSnapshotInfoByIdAndRole(long snapshotId, DataStoreRole dataStoreRole, Long zoneId) throws CloudRuntimeException { + SnapshotInfo snapInfo = null; + if (DataStoreRole.Primary.equals(dataStoreRole)) { + snapInfo = snapshotFactory.getSnapshotOnPrimaryStore(snapshotId); + } else { + snapInfo = snapshotFactory.getSnapshotWithRoleAndZone(snapshotId, dataStoreRole, zoneId); + } if (snapInfo != null) { return snapInfo; @@ -168,7 +178,7 @@ public boolean isKvmSnapshotOnlyInPrimaryStorage(Snapshot snapshot, DataStoreRol } public DataStoreRole getDataStoreRole(Snapshot snapshot) { - SnapshotDataStoreVO snapshotStore = snapshotDataStoreDao.findBySnapshot(snapshot.getId(), DataStoreRole.Primary); + SnapshotDataStoreVO snapshotStore = snapshotDataStoreDao.findOneBySnapshotAndDatastoreRole(snapshot.getId(), DataStoreRole.Primary); if (snapshotStore == null) { return DataStoreRole.Image; diff --git a/server/src/test/java/com/cloud/event/ActionEventUtilsTest.java b/server/src/test/java/com/cloud/event/ActionEventUtilsTest.java index 0249e4b63c4d..aed28702df5c 100644 --- a/server/src/test/java/com/cloud/event/ActionEventUtilsTest.java +++ b/server/src/test/java/com/cloud/event/ActionEventUtilsTest.java @@ -16,24 +16,15 @@ // under the License. package com.cloud.event; -import com.cloud.configuration.Config; -import com.cloud.event.dao.EventDao; -import com.cloud.network.IpAddress; -import com.cloud.projects.dao.ProjectDao; -import com.cloud.storage.Snapshot; -import com.cloud.storage.Volume; -import com.cloud.user.Account; -import com.cloud.user.AccountVO; -import com.cloud.user.User; -import com.cloud.user.UserVO; -import com.cloud.user.dao.AccountDao; -import com.cloud.user.dao.UserDao; -import com.cloud.utils.component.ComponentContext; -import com.cloud.utils.db.EntityManager; -import com.cloud.vm.VirtualMachine; -import com.cloud.vm.snapshot.VMSnapshot; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +import javax.inject.Inject; + import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; @@ -51,13 +42,23 @@ import org.mockito.junit.MockitoJUnitRunner; import org.mockito.stubbing.Answer; -import javax.inject.Inject; -import java.lang.reflect.Field; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.UUID; +import com.cloud.configuration.Config; +import com.cloud.event.dao.EventDao; +import com.cloud.network.IpAddress; +import com.cloud.projects.dao.ProjectDao; +import com.cloud.storage.Snapshot; +import com.cloud.user.Account; +import com.cloud.user.AccountVO; +import com.cloud.user.User; +import com.cloud.user.UserVO; +import com.cloud.user.dao.AccountDao; +import com.cloud.user.dao.UserDao; +import com.cloud.utils.component.ComponentContext; +import com.cloud.utils.db.EntityManager; +import com.cloud.vm.VirtualMachine; +import com.cloud.vm.snapshot.VMSnapshot; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; @RunWith(MockitoJUnitRunner.class) public class ActionEventUtilsTest { @@ -356,20 +357,13 @@ public void testSnapshotEventResource() { final Long snapshotResourceId = 100L; final String snapshotResourceType = ApiCommandResourceType.Snapshot.toString(); final String snapshotResourceUuid = UUID.randomUUID().toString(); - final Long resourceId = 1L; - final String resourceType = ApiCommandResourceType.Volume.toString(); - final String resourceUuid = UUID.randomUUID().toString(); Snapshot snapshot = Mockito.mock(Snapshot.class); Mockito.when(snapshot.getUuid()).thenReturn(snapshotResourceUuid); - Mockito.when(snapshot.getVolumeId()).thenReturn(resourceId); - Volume volume = Mockito.mock(Volume.class); - Mockito.when(volume.getUuid()).thenReturn(resourceUuid); Mockito.when(entityMgr.validEntityType(Snapshot.class)).thenReturn(true); Mockito.when(entityMgr.findByIdIncludingRemoved(Snapshot.class, snapshotResourceId)).thenReturn(snapshot); - Mockito.when(entityMgr.findByIdIncludingRemoved(Volume.class, resourceId)).thenReturn(volume); ActionEventUtils.onActionEvent(USER_ID, ACCOUNT_ID, account.getDomainId(), EventTypes.EVENT_SNAPSHOT_CREATE, "Test event", snapshotResourceId, snapshotResourceType); - checkEventResourceAndUnregisterContext(resourceId, resourceUuid, resourceType); + checkEventResourceAndUnregisterContext(snapshotResourceId, snapshotResourceUuid, snapshotResourceType); } @Test diff --git a/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java b/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java index 8940ef4da256..8ea14846bef2 100644 --- a/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java +++ b/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java @@ -16,6 +16,71 @@ // under the License. package com.cloud.storage; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyLong; +import static org.mockito.Matchers.anyString; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.ExecutionException; + +import org.apache.cloudstack.acl.ControlledEntity; +import org.apache.cloudstack.acl.SecurityChecker.AccessType; +import org.apache.cloudstack.api.command.user.volume.CreateVolumeCmd; +import org.apache.cloudstack.api.command.user.volume.DetachVolumeCmd; +import org.apache.cloudstack.api.command.user.volume.MigrateVolumeCmd; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; +import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeService; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeService.VolumeApiResult; +import org.apache.cloudstack.framework.async.AsyncCallFuture; +import org.apache.cloudstack.framework.jobs.AsyncJobExecutionContext; +import org.apache.cloudstack.framework.jobs.AsyncJobManager; +import org.apache.cloudstack.framework.jobs.dao.AsyncJobJoinMapDao; +import org.apache.cloudstack.framework.jobs.impl.AsyncJobVO; +import org.apache.cloudstack.resourcedetail.dao.SnapshotPolicyDetailsDao; +import org.apache.cloudstack.snapshot.SnapshotHelper; +import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; +import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO; +import org.apache.cloudstack.utils.bytescale.ByteScaleUtils; +import org.apache.commons.collections.CollectionUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.mockito.InOrder; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; +import org.springframework.test.util.ReflectionTestUtils; + import com.cloud.api.query.dao.ServiceOfferingJoinDao; import com.cloud.api.query.vo.ServiceOfferingJoinVO; import com.cloud.configuration.Resource; @@ -63,69 +128,6 @@ import com.cloud.vm.dao.VMInstanceDao; import com.cloud.vm.snapshot.VMSnapshotVO; import com.cloud.vm.snapshot.dao.VMSnapshotDao; -import org.apache.cloudstack.acl.ControlledEntity; -import org.apache.cloudstack.acl.SecurityChecker.AccessType; -import org.apache.cloudstack.api.command.user.volume.CreateVolumeCmd; -import org.apache.cloudstack.api.command.user.volume.DetachVolumeCmd; -import org.apache.cloudstack.api.command.user.volume.MigrateVolumeCmd; -import org.apache.cloudstack.context.CallContext; -import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; -import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; -import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStore; -import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; -import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory; -import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; -import org.apache.cloudstack.engine.subsystem.api.storage.VolumeService; -import org.apache.cloudstack.engine.subsystem.api.storage.VolumeService.VolumeApiResult; -import org.apache.cloudstack.framework.async.AsyncCallFuture; -import org.apache.cloudstack.framework.jobs.AsyncJobExecutionContext; -import org.apache.cloudstack.framework.jobs.AsyncJobManager; -import org.apache.cloudstack.framework.jobs.dao.AsyncJobJoinMapDao; -import org.apache.cloudstack.framework.jobs.impl.AsyncJobVO; -import org.apache.cloudstack.snapshot.SnapshotHelper; -import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; -import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; -import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; -import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; -import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreDao; -import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO; -import org.apache.cloudstack.utils.bytescale.ByteScaleUtils; -import org.apache.commons.collections.CollectionUtils; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.junit.runner.RunWith; -import org.mockito.InOrder; -import org.mockito.InjectMocks; -import org.mockito.Mock; -import org.mockito.MockedStatic; -import org.mockito.Mockito; -import org.mockito.Spy; -import org.mockito.junit.MockitoJUnitRunner; -import org.springframework.test.util.ReflectionTestUtils; - -import java.lang.reflect.Field; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.UUID; -import java.util.concurrent.ExecutionException; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.doNothing; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.lenient; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; @RunWith(MockitoJUnitRunner.class) public class VolumeApiServiceImplTest { @@ -219,6 +221,9 @@ public class VolumeApiServiceImplTest { @Mock private SnapshotDao snapshotDaoMock; + @Mock + private SnapshotPolicyDetailsDao snapshotPolicyDetailsDao; + @Mock private Project projectMock; @@ -531,7 +536,7 @@ public void testTakeSnapshotF1() throws ResourceAllocationException { when(volumeDataFactoryMock.getVolume(anyLong())).thenReturn(volumeInfoMock); when(volumeInfoMock.getState()).thenReturn(Volume.State.Allocated); lenient().when(volumeInfoMock.getPoolId()).thenReturn(1L); - volumeApiServiceImpl.takeSnapshot(5L, Snapshot.MANUAL_POLICY_ID, 3L, null, false, null, false, null); + volumeApiServiceImpl.takeSnapshot(5L, Snapshot.MANUAL_POLICY_ID, 3L, null, false, null, false, null, null); } @Test @@ -544,7 +549,7 @@ public void testTakeSnapshotF2() throws ResourceAllocationException { final TaggedResourceService taggedResourceService = Mockito.mock(TaggedResourceService.class); Mockito.lenient().when(taggedResourceService.createTags(any(), any(), any(), any())).thenReturn(null); ReflectionTestUtils.setField(volumeApiServiceImpl, "taggedResourceService", taggedResourceService); - volumeApiServiceImpl.takeSnapshot(5L, Snapshot.MANUAL_POLICY_ID, 3L, null, false, null, false, null); + volumeApiServiceImpl.takeSnapshot(5L, Snapshot.MANUAL_POLICY_ID, 3L, null, false, null, false, null, null); } @Test @@ -592,7 +597,7 @@ public void testUpdateMissingRootDiskControllerWithValidChainInfo() { @Test public void testAllocSnapshotNonManagedStorageArchive() { try { - volumeApiServiceImpl.allocSnapshot(6L, 1L, "test", Snapshot.LocationType.SECONDARY); + volumeApiServiceImpl.allocSnapshot(6L, 1L, "test", Snapshot.LocationType.SECONDARY, null); } catch (InvalidParameterValueException e) { Assert.assertEquals(e.getMessage(), "VolumeId: 6 LocationType is supported only for managed storage"); return; diff --git a/server/src/test/java/com/cloud/storage/snapshot/SnapshotManagerImplTest.java b/server/src/test/java/com/cloud/storage/snapshot/SnapshotManagerImplTest.java new file mode 100644 index 000000000000..1d7cf0a50d24 --- /dev/null +++ b/server/src/test/java/com/cloud/storage/snapshot/SnapshotManagerImplTest.java @@ -0,0 +1,408 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.storage.snapshot; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutionException; + +import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotResult; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotService; +import org.apache.cloudstack.framework.async.AsyncCallFuture; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.stubbing.Answer; + +import com.cloud.dc.DataCenter; +import com.cloud.dc.DataCenterVO; +import com.cloud.dc.dao.DataCenterDao; +import com.cloud.event.ActionEventUtils; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.exception.PermissionDeniedException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.org.Grouping; +import com.cloud.storage.DataStoreRole; +import com.cloud.storage.Snapshot; +import com.cloud.storage.SnapshotVO; +import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.SnapshotDao; +import com.cloud.storage.dao.SnapshotZoneDao; +import com.cloud.storage.dao.VolumeDao; +import com.cloud.user.Account; +import com.cloud.user.AccountManager; +import com.cloud.user.AccountVO; +import com.cloud.user.ResourceLimitService; +import com.cloud.user.dao.AccountDao; +import com.cloud.utils.Pair; + +@RunWith(MockitoJUnitRunner.class) +public class SnapshotManagerImplTest { + @Mock + AccountDao accountDao; + @Mock + SnapshotDao snapshotDao; + @Mock + AccountManager accountManager; + @Mock + SnapshotService snapshotService; + @Mock + SnapshotDataFactory snapshotDataFactory; + @Mock + ResourceLimitService resourceLimitService; + @Mock + DataCenterDao dataCenterDao; + @Mock + SnapshotDataStoreDao snapshotStoreDao; + @Mock + DataStoreManager dataStoreManager; + @Mock + SnapshotZoneDao snapshotZoneDao; + @Mock + VolumeDao volumeDao; + @InjectMocks + SnapshotManagerImpl snapshotManager = new SnapshotManagerImpl(); + + @Test + public void testGetSnapshotZoneImageStoreValid() { + final long snapshotId = 1L; + final long zoneId = 1L; + final long storeId = 1L; + SnapshotDataStoreVO ref = Mockito.mock(SnapshotDataStoreVO.class); + Mockito.when(ref.getDataStoreId()).thenReturn(storeId); + Mockito.when(ref.getRole()).thenReturn(DataStoreRole.Image); + List snapshotStoreList = List.of(Mockito.mock(SnapshotDataStoreVO.class), ref); + Mockito.when(dataStoreManager.getStoreZoneId(storeId, DataStoreRole.Image)).thenReturn(zoneId); + Mockito.when(dataStoreManager.getDataStore(storeId, DataStoreRole.Image)).thenReturn(Mockito.mock(DataStore.class)); + Mockito.when(snapshotStoreDao.listReadyBySnapshot(snapshotId, DataStoreRole.Image)).thenReturn(snapshotStoreList); + DataStore store = snapshotManager.getSnapshotZoneImageStore(snapshotId, zoneId); + Assert.assertNotNull(store); + } + + @Test + public void testGetSnapshotZoneImageStoreNull() { + final long snapshotId = 1L; + final long zoneId = 1L; + final long storeId = 1L; + SnapshotDataStoreVO ref = Mockito.mock(SnapshotDataStoreVO.class); + Mockito.when(ref.getDataStoreId()).thenReturn(storeId); + Mockito.when(ref.getRole()).thenReturn(DataStoreRole.Image); + List snapshotStoreList = List.of(ref); + Mockito.when(dataStoreManager.getStoreZoneId(storeId, DataStoreRole.Image)).thenReturn(100L); + Mockito.when(snapshotStoreDao.listReadyBySnapshot(snapshotId, DataStoreRole.Image)).thenReturn(snapshotStoreList); + DataStore store = snapshotManager.getSnapshotZoneImageStore(snapshotId, zoneId); + Assert.assertNull(store); + } + + @Test(expected = InvalidParameterValueException.class) + public void testGetStoreRefsAndZonesForSnapshotDeleteException() { + final long snapshotId = 1L; + final long zoneId = 1L; + Mockito.when(dataCenterDao.findById(zoneId)).thenReturn(null); + snapshotManager.getStoreRefsAndZonesForSnapshotDelete(snapshotId, zoneId); + } + + @Test + public void testGetStoreRefsAndZonesForSnapshotDeleteMultiZones() { + final long snapshotId = 1L; + SnapshotDataStoreVO ref = Mockito.mock(SnapshotDataStoreVO.class); + Mockito.when(ref.getDataStoreId()).thenReturn(1L); + Mockito.when(ref.getRole()).thenReturn(DataStoreRole.Image); + SnapshotDataStoreVO ref1 = Mockito.mock(SnapshotDataStoreVO.class); + Mockito.when(ref1.getDataStoreId()).thenReturn(2L); + Mockito.when(ref1.getRole()).thenReturn(DataStoreRole.Image); + List snapshotStoreList = List.of(ref, ref1); + Mockito.when(snapshotStoreDao.findBySnapshotId(snapshotId)).thenReturn(snapshotStoreList); + Mockito.when(dataStoreManager.getStoreZoneId(1L, DataStoreRole.Image)).thenReturn(100L); + Mockito.when(dataStoreManager.getStoreZoneId(2L, DataStoreRole.Image)).thenReturn(101L); + Pair, List> pair = snapshotManager.getStoreRefsAndZonesForSnapshotDelete(snapshotId, null); + Assert.assertNotNull(pair.first()); + Assert.assertNotNull(pair.second()); + Assert.assertEquals(snapshotStoreList.size(), pair.first().size()); + Assert.assertEquals(2, pair.second().size()); + } + + @Test + public void testGetStoreRefsAndZonesForSnapshotDeleteSingle() { + final long snapshotId = 1L; + final long zoneId = 1L; + Mockito.when(dataCenterDao.findById(zoneId)).thenReturn(Mockito.mock(DataCenterVO.class)); + SnapshotDataStoreVO ref = Mockito.mock(SnapshotDataStoreVO.class); + Mockito.when(ref.getDataStoreId()).thenReturn(1L); + Mockito.when(ref.getRole()).thenReturn(DataStoreRole.Image); + SnapshotDataStoreVO ref1 = Mockito.mock(SnapshotDataStoreVO.class); + Mockito.when(ref1.getDataStoreId()).thenReturn(2L); + Mockito.when(ref1.getRole()).thenReturn(DataStoreRole.Primary); + SnapshotDataStoreVO ref2 = Mockito.mock(SnapshotDataStoreVO.class); + Mockito.when(ref2.getDataStoreId()).thenReturn(3L); + Mockito.when(ref2.getRole()).thenReturn(DataStoreRole.Image); + List snapshotStoreList = List.of(ref, ref1, ref2); + Mockito.when(snapshotStoreDao.findBySnapshotId(snapshotId)).thenReturn(snapshotStoreList); + Mockito.when(dataStoreManager.getStoreZoneId(1L, DataStoreRole.Image)).thenReturn(zoneId); + Mockito.when(dataStoreManager.getStoreZoneId(2L, DataStoreRole.Primary)).thenReturn(zoneId); + Mockito.when(dataStoreManager.getStoreZoneId(3L, DataStoreRole.Image)).thenReturn(2L); + Pair, List> pair = snapshotManager.getStoreRefsAndZonesForSnapshotDelete(snapshotId, zoneId); + Assert.assertNotNull(pair.first()); + Assert.assertNotNull(pair.second()); + Assert.assertEquals(snapshotStoreList.size() - 1, pair.first().size()); + Assert.assertEquals(1, pair.second().size()); + } + @Test + public void testValidatePolicyZonesNoZones() { + snapshotManager.validatePolicyZones(null, Mockito.mock(VolumeVO.class), Mockito.mock(Account.class)); + } + + @Test(expected = InvalidParameterValueException.class) + public void testValidatePolicyZonesVolumeEdgeZone() { + VolumeVO volumeVO = Mockito.mock(VolumeVO.class); + Mockito.when(volumeVO.getDataCenterId()).thenReturn(1L); + DataCenterVO zone = Mockito.mock(DataCenterVO.class); + Mockito.when(zone.getType()).thenReturn(DataCenter.Type.Edge); + Mockito.when(dataCenterDao.findById(1L)).thenReturn(zone); + snapshotManager.validatePolicyZones(List.of(1L), volumeVO, Mockito.mock(Account.class)); + } + + @Test(expected = InvalidParameterValueException.class) + public void testValidatePolicyZonesNullZone() { + VolumeVO volumeVO = Mockito.mock(VolumeVO.class); + Mockito.when(volumeVO.getDataCenterId()).thenReturn(1L); + DataCenterVO zone = Mockito.mock(DataCenterVO.class); + Mockito.when(zone.getType()).thenReturn(DataCenter.Type.Core); + Mockito.when(dataCenterDao.findById(1L)).thenReturn(zone); + Mockito.when(dataCenterDao.findById(2L)).thenReturn(null); + snapshotManager.validatePolicyZones(List.of(2L), volumeVO, Mockito.mock(Account.class)); + } + + @Test(expected = PermissionDeniedException.class) + public void testValidatePolicyZonesDisabledZone() { + VolumeVO volumeVO = Mockito.mock(VolumeVO.class); + Mockito.when(volumeVO.getDataCenterId()).thenReturn(1L); + DataCenterVO zone = Mockito.mock(DataCenterVO.class); + Mockito.when(zone.getType()).thenReturn(DataCenter.Type.Core); + Mockito.when(dataCenterDao.findById(1L)).thenReturn(zone); + DataCenterVO zone1 = Mockito.mock(DataCenterVO.class); + Mockito.when(zone1.getAllocationState()).thenReturn(Grouping.AllocationState.Disabled); + Mockito.when(dataCenterDao.findById(2L)).thenReturn(zone1); + Mockito.when(accountManager.isRootAdmin(Mockito.any())).thenReturn(false); + snapshotManager.validatePolicyZones(List.of(2L), volumeVO, Mockito.mock(Account.class)); + } + + @Test(expected = InvalidParameterValueException.class) + public void testValidatePolicyZonesEdgeZone() { + VolumeVO volumeVO = Mockito.mock(VolumeVO.class); + Mockito.when(volumeVO.getDataCenterId()).thenReturn(1L); + DataCenterVO zone = Mockito.mock(DataCenterVO.class); + Mockito.when(zone.getType()).thenReturn(DataCenter.Type.Core); + Mockito.when(dataCenterDao.findById(1L)).thenReturn(zone); + DataCenterVO zone1 = Mockito.mock(DataCenterVO.class); + Mockito.when(zone1.getType()).thenReturn(DataCenter.Type.Edge); + Mockito.when(zone1.getAllocationState()).thenReturn(Grouping.AllocationState.Enabled); + Mockito.when(dataCenterDao.findById(2L)).thenReturn(zone1); + snapshotManager.validatePolicyZones(List.of(2L), volumeVO, Mockito.mock(Account.class)); + } + + @Test + public void testValidatePolicyZonesValidZone() { + VolumeVO volumeVO = Mockito.mock(VolumeVO.class); + Mockito.when(volumeVO.getDataCenterId()).thenReturn(1L); + DataCenterVO zone = Mockito.mock(DataCenterVO.class); + Mockito.when(zone.getType()).thenReturn(DataCenter.Type.Core); + Mockito.when(dataCenterDao.findById(1L)).thenReturn(zone); + DataCenterVO zone1 = Mockito.mock(DataCenterVO.class); + Mockito.when(zone1.getType()).thenReturn(DataCenter.Type.Core); + Mockito.when(zone1.getAllocationState()).thenReturn(Grouping.AllocationState.Enabled); + Mockito.when(dataCenterDao.findById(2L)).thenReturn(zone1); + snapshotManager.validatePolicyZones(List.of(2L), volumeVO, Mockito.mock(Account.class)); + } + + @Test + public void testCopyNewSnapshotToZonesNoZones() { + snapshotManager.copyNewSnapshotToZones(1L, 1L, new ArrayList<>()); + } + + @Test + public void testCopyNewSnapshotToZones() { + final long snapshotId = 1L; + SnapshotVO snapshotVO = Mockito.mock(SnapshotVO.class); + Mockito.when(snapshotVO.getId()).thenReturn(snapshotId); + Mockito.when(snapshotVO.getAccountId()).thenReturn(1L); + Mockito.when(snapshotDao.findById(snapshotId)).thenReturn(snapshotVO); + final long zoneId = 1L; + final long storeId = 1L; + final long destZoneId = 2L; + DataCenterVO zone = Mockito.mock(DataCenterVO.class); + Mockito.when(zone.getId()).thenReturn(destZoneId); + Mockito.when(dataCenterDao.findById(destZoneId)).thenReturn(zone); + SnapshotDataStoreVO ref = Mockito.mock(SnapshotDataStoreVO.class); + Mockito.when(ref.getDataStoreId()).thenReturn(storeId); + Mockito.when(ref.getRole()).thenReturn(DataStoreRole.Image); + List snapshotStoreList = List.of(Mockito.mock(SnapshotDataStoreVO.class), ref); + Mockito.when(dataStoreManager.getStoreZoneId(storeId, DataStoreRole.Image)).thenReturn(zoneId); + DataStore store = Mockito.mock(DataStore.class); + Mockito.when(store.getId()).thenReturn(storeId); + Mockito.when(dataStoreManager.getDataStore(storeId, DataStoreRole.Image)).thenReturn(store); + Mockito.when(snapshotStoreDao.listReadyBySnapshot(snapshotId, DataStoreRole.Image)).thenReturn(snapshotStoreList); + Mockito.when(snapshotDataFactory.getSnapshot(Mockito.anyLong(), Mockito.any())).thenReturn(Mockito.mock(SnapshotInfo.class)); + CreateCmdResult result = Mockito.mock(CreateCmdResult.class); + Mockito.when(result.isFailed()).thenReturn(false); + Mockito.when(result.getPath()).thenReturn("SOMEPATH"); + AsyncCallFuture future = Mockito.mock(AsyncCallFuture.class); + Mockito.when(dataStoreManager.getImageStoresByScopeExcludingReadOnly(Mockito.any())).thenReturn(List.of(Mockito.mock(DataStore.class))); + Mockito.when(dataStoreManager.getImageStoreWithFreeCapacity(Mockito.anyList())).thenReturn(Mockito.mock(DataStore.class)); + Mockito.when(snapshotStoreDao.findByStoreSnapshot(DataStoreRole.Image, 1L, 1L)).thenReturn(Mockito.mock(SnapshotDataStoreVO.class)); + AccountVO account = Mockito.mock(AccountVO.class); + Mockito.when(account.getId()).thenReturn(1L); + Mockito.when(accountDao.findById(Mockito.anyLong())).thenReturn(account); + SnapshotResult result1 = Mockito.mock(SnapshotResult.class); + Mockito.when(result1.isFailed()).thenReturn(false); + AsyncCallFuture future1 = Mockito.mock(AsyncCallFuture.class); + try { + Mockito.doNothing().when(resourceLimitService).checkResourceLimit(Mockito.any(), Mockito.any(), Mockito.anyLong()); + Mockito.when(future.get()).thenReturn(result); + Mockito.when(snapshotService.queryCopySnapshot(Mockito.any())).thenReturn(future); + Mockito.when(future1.get()).thenReturn(result1); + Mockito.when(snapshotService.copySnapshot(Mockito.any(SnapshotInfo.class), Mockito.anyString(), Mockito.any(DataStore.class))).thenReturn(future1); + } catch (ResourceAllocationException | ResourceUnavailableException | ExecutionException | InterruptedException e) { + Assert.fail(e.getMessage()); + } + List addedZone = new ArrayList<>(); + Mockito.doAnswer((Answer) invocation -> { + Long zoneId1 = (Long) invocation.getArguments()[1]; + addedZone.add(zoneId1); + return null; + }).when(snapshotZoneDao).addSnapshotToZone(Mockito.anyLong(), Mockito.anyLong()); + try (MockedStatic utilities = Mockito.mockStatic(ActionEventUtils.class)) { + utilities.when(() -> ActionEventUtils.onStartedActionEvent(Mockito.anyLong(), Mockito.anyLong(), Mockito.anyString(), + Mockito.anyString(), Mockito.anyLong(), Mockito.anyString(), Mockito.anyBoolean(), Mockito.anyLong())).thenReturn(1L); + snapshotManager.copyNewSnapshotToZones(snapshotId, 1L, List.of(2L)); + Assert.assertEquals(1, addedZone.size()); + } + } + + @Test(expected = InvalidParameterValueException.class) + public void testGetCheckedSnapshotForCopyNoSnapshot() { + snapshotManager.getCheckedSnapshotForCopy(1L, List.of(100L), null); + } + + @Test(expected = InvalidParameterValueException.class) + public void testGetCheckedSnapshotForCopyNoSnapshotBackup() { + final long snapshotId = 1L; + SnapshotVO snapshotVO = Mockito.mock(SnapshotVO.class); + Mockito.when(snapshotDao.findById(snapshotId)).thenReturn(snapshotVO); + snapshotManager.getCheckedSnapshotForCopy(snapshotId, List.of(100L), null); + } + + @Test(expected = InvalidParameterValueException.class) + public void testGetCheckedSnapshotForCopyNotOnSecondary() { + final long snapshotId = 1L; + SnapshotVO snapshotVO = Mockito.mock(SnapshotVO.class); + Mockito.when(snapshotVO.getState()).thenReturn(Snapshot.State.BackedUp); + Mockito.when(snapshotVO.getLocationType()).thenReturn(Snapshot.LocationType.PRIMARY); + Mockito.when(snapshotDao.findById(snapshotId)).thenReturn(snapshotVO); + snapshotManager.getCheckedSnapshotForCopy(snapshotId, List.of(100L), null); + } + + @Test(expected = InvalidParameterValueException.class) + public void testGetCheckedSnapshotForCopyDestNotSpecified() { + final long snapshotId = 1L; + SnapshotVO snapshotVO = Mockito.mock(SnapshotVO.class); + Mockito.when(snapshotVO.getState()).thenReturn(Snapshot.State.BackedUp); + Mockito.when(snapshotDao.findById(snapshotId)).thenReturn(snapshotVO); + snapshotManager.getCheckedSnapshotForCopy(snapshotId, new ArrayList<>(), null); + } + + @Test(expected = InvalidParameterValueException.class) + public void testGetCheckedSnapshotForCopyDestContainsSource() { + final long snapshotId = 1L; + SnapshotVO snapshotVO = Mockito.mock(SnapshotVO.class); + Mockito.when(snapshotVO.getState()).thenReturn(Snapshot.State.BackedUp); + Mockito.when(snapshotVO.getVolumeId()).thenReturn(1L); + Mockito.when(snapshotDao.findById(snapshotId)).thenReturn(snapshotVO); + Mockito.when(volumeDao.findById(Mockito.anyLong())).thenReturn(Mockito.mock(VolumeVO.class)); + snapshotManager.getCheckedSnapshotForCopy(snapshotId, List.of(100L, 1L), 1L); + } + + @Test(expected = InvalidParameterValueException.class) + public void testGetCheckedSnapshotForCopyNullSourceZone() { + final long snapshotId = 1L; + SnapshotVO snapshotVO = Mockito.mock(SnapshotVO.class); + Mockito.when(snapshotVO.getState()).thenReturn(Snapshot.State.BackedUp); + Mockito.when(snapshotVO.getVolumeId()).thenReturn(1L); + Mockito.when(snapshotDao.findById(snapshotId)).thenReturn(snapshotVO); + VolumeVO volumeVO = Mockito.mock(VolumeVO.class); + Mockito.when(volumeVO.getDataCenterId()).thenReturn(1L); + Mockito.when(volumeDao.findById(Mockito.anyLong())).thenReturn(volumeVO); + snapshotManager.getCheckedSnapshotForCopy(snapshotId, List.of(100L, 101L), null); + } + + @Test + public void testGetCheckedSnapshotForCopyValid() { + final long snapshotId = 1L; + final Long zoneId = 1L; + SnapshotVO snapshotVO = Mockito.mock(SnapshotVO.class); + Mockito.when(snapshotVO.getState()).thenReturn(Snapshot.State.BackedUp); + Mockito.when(snapshotVO.getVolumeId()).thenReturn(1L); + Mockito.when(snapshotDao.findById(snapshotId)).thenReturn(snapshotVO); + VolumeVO volumeVO = Mockito.mock(VolumeVO.class); + Mockito.when(volumeVO.getDataCenterId()).thenReturn(zoneId); + Mockito.when(volumeDao.findById(Mockito.anyLong())).thenReturn(volumeVO); + Mockito.when(dataCenterDao.findById(zoneId)).thenReturn(Mockito.mock(DataCenterVO.class)); + Pair result = snapshotManager.getCheckedSnapshotForCopy(snapshotId, List.of(100L, 101L), null); + Assert.assertNotNull(result.first()); + Assert.assertEquals(zoneId, result.second()); + } + + @Test + public void testGetCheckedSnapshotForCopyNullDest() { + final long snapshotId = 1L; + final Long zoneId = 1L; + SnapshotVO snapshotVO = Mockito.mock(SnapshotVO.class); + Mockito.when(snapshotVO.getState()).thenReturn(Snapshot.State.BackedUp); + Mockito.when(snapshotVO.getVolumeId()).thenReturn(1L); + Mockito.when(snapshotDao.findById(snapshotId)).thenReturn(snapshotVO); + VolumeVO volumeVO = Mockito.mock(VolumeVO.class); + Mockito.when(volumeVO.getDataCenterId()).thenReturn(zoneId); + Mockito.when(volumeDao.findById(Mockito.anyLong())).thenReturn(volumeVO); + Mockito.when(dataCenterDao.findById(zoneId)).thenReturn(Mockito.mock(DataCenterVO.class)); + Pair result = snapshotManager.getCheckedSnapshotForCopy(snapshotId, List.of(100L, 101L), null); + Assert.assertNotNull(result.first()); + Assert.assertEquals(zoneId, result.second()); + } + + @Test + public void testGetCheckedDestinationZoneForSnapshotCopy() { + long zoneId = 1L; + DataCenterVO dataCenterVO = Mockito.mock(DataCenterVO.class); + Mockito.when(dataCenterVO.getAllocationState()).thenReturn(Grouping.AllocationState.Enabled); + Mockito.when(dataCenterVO.getType()).thenReturn(DataCenter.Type.Core); + Mockito.when(dataCenterDao.findById(zoneId)).thenReturn(dataCenterVO); + Assert.assertNotNull(snapshotManager.getCheckedDestinationZoneForSnapshotCopy(zoneId, false)); + } +} diff --git a/server/src/test/java/com/cloud/storage/snapshot/SnapshotManagerTest.java b/server/src/test/java/com/cloud/storage/snapshot/SnapshotManagerTest.java index 1bb5801cd135..fb7319bd46b9 100755 --- a/server/src/test/java/com/cloud/storage/snapshot/SnapshotManagerTest.java +++ b/server/src/test/java/com/cloud/storage/snapshot/SnapshotManagerTest.java @@ -16,6 +16,52 @@ // under the License. package com.cloud.storage.snapshot; +import static org.mockito.ArgumentMatchers.nullable; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyLong; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotService; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotStrategy; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotStrategy.SnapshotOperation; +import org.apache.cloudstack.engine.subsystem.api.storage.StorageStrategyFactory; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.snapshot.SnapshotHelper; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.BDDMockito; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.stubbing.Answer; + import com.cloud.configuration.Resource.ResourceType; import com.cloud.dc.DataCenter; import com.cloud.dc.DataCenterVO; @@ -25,6 +71,7 @@ import com.cloud.hypervisor.Hypervisor; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.resource.ResourceManager; +import com.cloud.server.ResourceTag; import com.cloud.server.TaggedResourceService; import com.cloud.storage.DataStoreRole; import com.cloud.storage.ScopeType; @@ -52,55 +99,11 @@ import com.cloud.vm.snapshot.VMSnapshot; import com.cloud.vm.snapshot.VMSnapshotVO; import com.cloud.vm.snapshot.dao.VMSnapshotDao; -import org.apache.cloudstack.context.CallContext; -import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; -import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; -import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory; -import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; -import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotService; -import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotStrategy; -import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotStrategy.SnapshotOperation; -import org.apache.cloudstack.engine.subsystem.api.storage.StorageStrategyFactory; -import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory; -import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; -import org.apache.cloudstack.framework.config.ConfigKey; -import org.apache.cloudstack.snapshot.SnapshotHelper; -import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; -import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; -import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; -import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.BDDMockito; -import org.mockito.Mock; -import org.mockito.MockedStatic; -import org.mockito.Mockito; -import org.mockito.MockitoAnnotations; -import org.mockito.Spy; -import org.mockito.junit.MockitoJUnitRunner; -import org.mockito.verification.VerificationMode; - -import java.lang.reflect.Field; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.UUID; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.ArgumentMatchers.nullable; -import static org.mockito.Mockito.doNothing; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; @RunWith(MockitoJUnitRunner.class) public class SnapshotManagerTest { - @Spy + + @InjectMocks SnapshotManagerImpl _snapshotMgr = new SnapshotManagerImpl(); @Mock SnapshotDao _snapshotDao; @@ -189,48 +192,29 @@ public class SnapshotManagerTest { private static final boolean TEST_SNAPSHOT_POLICY_DISPLAY = true; private static final boolean TEST_SNAPSHOT_POLICY_ACTIVE = true; - private AutoCloseable closeable; - @Before public void setup() throws ResourceAllocationException { - closeable = MockitoAnnotations.openMocks(this); - _snapshotMgr._snapshotDao = _snapshotDao; - _snapshotMgr._volsDao = _volumeDao; - _snapshotMgr._vmDao = _vmDao; - _snapshotMgr.volFactory = volumeFactory; - _snapshotMgr.snapshotFactory = snapshotFactory; - _snapshotMgr._storageStrategyFactory = _storageStrategyFactory; - _snapshotMgr._accountMgr = _accountMgr; - _snapshotMgr._resourceLimitMgr = _resourceLimitMgr; - _snapshotMgr._storagePoolDao = _storagePoolDao; - _snapshotMgr._resourceMgr = _resourceMgr; - _snapshotMgr._vmSnapshotDao = _vmSnapshotDao; - _snapshotMgr._snapshotStoreDao = snapshotStoreDao; - _snapshotMgr.snapshotHelper = snapshotHelperMock; - _snapshotMgr._snapshotPolicyDao = snapshotPolicyDaoMock; - _snapshotMgr._snapSchedMgr = snapshotSchedulerMock; - _snapshotMgr.taggedResourceService = taggedResourceServiceMock; - _snapshotMgr.dataCenterDao = dataCenterDao; when(_snapshotDao.findById(anyLong())).thenReturn(snapshotMock); when(snapshotMock.getVolumeId()).thenReturn(TEST_VOLUME_ID); when(_volumeDao.findById(anyLong())).thenReturn(volumeMock); when(volumeMock.getState()).thenReturn(Volume.State.Ready); + when(volumeMock.getId()).thenReturn(TEST_VOLUME_ID); when(volumeFactory.getVolume(anyLong())).thenReturn(volumeInfoMock); when(volumeInfoMock.getDataStore()).thenReturn(storeMock); when(volumeInfoMock.getState()).thenReturn(Volume.State.Ready); when(storeMock.getId()).thenReturn(TEST_STORAGE_POOL_ID); - when(snapshotFactory.getSnapshot(anyLong(), any(DataStoreRole.class))).thenReturn(snapshotInfoMock); - when(_storageStrategyFactory.getSnapshotStrategy(any(SnapshotVO.class), Mockito.eq(SnapshotOperation.BACKUP))).thenReturn(snapshotStrategy); - when(_storageStrategyFactory.getSnapshotStrategy(any(SnapshotVO.class), Mockito.eq(SnapshotOperation.REVERT))).thenReturn(snapshotStrategy); + when(snapshotFactory.getSnapshotWithRoleAndZone(anyLong(), Mockito.any(DataStoreRole.class), Mockito.anyLong())).thenReturn(snapshotInfoMock); + when(_storageStrategyFactory.getSnapshotStrategy(Mockito.any(SnapshotVO.class), Mockito.eq(SnapshotOperation.BACKUP))).thenReturn(snapshotStrategy); + when(_storageStrategyFactory.getSnapshotStrategy(Mockito.any(SnapshotVO.class), Mockito.eq(SnapshotOperation.REVERT))).thenReturn(snapshotStrategy); - doNothing().when(_snapshotMgr._resourceLimitMgr).checkResourceLimit(any(Account.class), any(ResourceType.class)); - doNothing().when(_snapshotMgr._resourceLimitMgr).checkResourceLimit(any(Account.class), any(ResourceType.class), anyLong()); - doNothing().when(_snapshotMgr._resourceLimitMgr).decrementResourceCount(anyLong(), any(ResourceType.class), anyLong()); - doNothing().when(_snapshotMgr._resourceLimitMgr).incrementResourceCount(anyLong(), any(ResourceType.class)); - doNothing().when(_snapshotMgr._resourceLimitMgr).incrementResourceCount(anyLong(), any(ResourceType.class), anyLong()); + doNothing().when(_resourceLimitMgr).checkResourceLimit(any(Account.class), any(ResourceType.class)); + doNothing().when(_resourceLimitMgr).checkResourceLimit(any(Account.class), any(ResourceType.class), anyLong()); + doNothing().when(_resourceLimitMgr).decrementResourceCount(anyLong(), any(ResourceType.class), anyLong()); + doNothing().when(_resourceLimitMgr).incrementResourceCount(anyLong(), any(ResourceType.class)); + doNothing().when(_resourceLimitMgr).incrementResourceCount(anyLong(), any(ResourceType.class), anyLong()); Account account = new AccountVO("testaccount", 1L, "networkdomain", Account.Type.NORMAL, "uuid"); UserVO user = new UserVO(1, "testuser", "password", "firstname", "lastName", "email", "timezone", UUID.randomUUID().toString(), User.Source.UNKNOWN); @@ -249,7 +233,6 @@ public void setup() throws ResourceAllocationException { @After public void tearDown() throws Exception { CallContext.unregister(); - closeable.close(); } // vm is destroyed @@ -312,10 +295,10 @@ public void testAllocSnapshotF4() throws ResourceAllocationException { } @Test(expected = InvalidParameterValueException.class) - public void testDeleteSnapshotF1() { + public void testDeleteSnapshotDestroyedFailure() { + when(_snapshotDao.findById(TEST_SNAPSHOT_ID)).thenReturn(snapshotMock); when(snapshotMock.getState()).thenReturn(Snapshot.State.Destroyed); - - _snapshotMgr.deleteSnapshot(TEST_SNAPSHOT_ID); + _snapshotMgr.deleteSnapshot(TEST_SNAPSHOT_ID, null); } // vm state not stopped @@ -390,22 +373,26 @@ public void testBackupSnapshotFromVmSnapshotF3() { @Test(expected = CloudRuntimeException.class) public void testArchiveSnapshotSnapshotNotOnPrimary() { - when(snapshotFactory.getSnapshot(anyLong(), Mockito.eq(DataStoreRole.Primary))).thenReturn(null); + when(snapshotFactory.getSnapshotOnPrimaryStore(anyLong())).thenReturn(null); _snapshotMgr.archiveSnapshot(TEST_SNAPSHOT_ID); } @Test(expected = CloudRuntimeException.class) public void testArchiveSnapshotSnapshotNotReady() { - when(snapshotFactory.getSnapshot(anyLong(), Mockito.eq(DataStoreRole.Primary))).thenReturn(snapshotInfoMock); + when(snapshotFactory.getSnapshotOnPrimaryStore(anyLong())).thenReturn(snapshotInfoMock); when(snapshotInfoMock.getStatus()).thenReturn(ObjectInDataStoreStateMachine.State.Destroyed); _snapshotMgr.archiveSnapshot(TEST_SNAPSHOT_ID); } - public void assertSnapshotPolicyResultAgainstPreBuiltInstance(SnapshotPolicyVO snapshotPolicyVo){ + public void assertSnapshotPolicyResultAgainstPreBuiltInstance(SnapshotPolicyVO snapshotPolicyVo, Short interval){ Assert.assertEquals(snapshotPolicyVoInstance.getVolumeId(), snapshotPolicyVo.getVolumeId()); Assert.assertEquals(snapshotPolicyVoInstance.getSchedule(), snapshotPolicyVo.getSchedule()); Assert.assertEquals(snapshotPolicyVoInstance.getTimezone(), snapshotPolicyVo.getTimezone()); - Assert.assertEquals(snapshotPolicyVoInstance.getInterval(), snapshotPolicyVo.getInterval()); + if (interval != null) { + Assert.assertEquals(interval.shortValue(), snapshotPolicyVo.getInterval()); + } else { + Assert.assertEquals(snapshotPolicyVoInstance.getInterval(), snapshotPolicyVo.getInterval()); + } Assert.assertEquals(snapshotPolicyVoInstance.getMaxSnaps(), snapshotPolicyVo.getMaxSnaps()); Assert.assertEquals(snapshotPolicyVoInstance.isDisplay(), snapshotPolicyVo.isDisplay()); Assert.assertEquals(snapshotPolicyVoInstance.isActive(), snapshotPolicyVo.isActive()); @@ -417,9 +404,9 @@ public void validateCreateSnapshotPolicy(){ Mockito.doReturn(null).when(snapshotSchedulerMock).scheduleNextSnapshotJob(any()); SnapshotPolicyVO result = _snapshotMgr.createSnapshotPolicy(TEST_VOLUME_ID, TEST_SNAPSHOT_POLICY_SCHEDULE, TEST_SNAPSHOT_POLICY_TIMEZONE, TEST_SNAPSHOT_POLICY_INTERVAL, - TEST_SNAPSHOT_POLICY_MAX_SNAPS, TEST_SNAPSHOT_POLICY_DISPLAY); + TEST_SNAPSHOT_POLICY_MAX_SNAPS, TEST_SNAPSHOT_POLICY_DISPLAY, null); - assertSnapshotPolicyResultAgainstPreBuiltInstance(result); + assertSnapshotPolicyResultAgainstPreBuiltInstance(result, null); } @Test @@ -432,9 +419,9 @@ public void validateUpdateSnapshotPolicy(){ TEST_SNAPSHOT_POLICY_MAX_SNAPS, TEST_SNAPSHOT_POLICY_DISPLAY); _snapshotMgr.updateSnapshotPolicy(snapshotPolicyVo, TEST_SNAPSHOT_POLICY_SCHEDULE, TEST_SNAPSHOT_POLICY_TIMEZONE, - TEST_SNAPSHOT_POLICY_INTERVAL, TEST_SNAPSHOT_POLICY_MAX_SNAPS, TEST_SNAPSHOT_POLICY_DISPLAY, TEST_SNAPSHOT_POLICY_ACTIVE); + TEST_SNAPSHOT_POLICY_INTERVAL, TEST_SNAPSHOT_POLICY_MAX_SNAPS, TEST_SNAPSHOT_POLICY_DISPLAY, TEST_SNAPSHOT_POLICY_ACTIVE, null); - assertSnapshotPolicyResultAgainstPreBuiltInstance(snapshotPolicyVo); + assertSnapshotPolicyResultAgainstPreBuiltInstance(snapshotPolicyVo, null); } @Test @@ -467,63 +454,50 @@ public void validatePersistSnapshotPolicyLockIsNotAquiredMustThrowException() { Mockito.doReturn(false).when(globalLockMock).lock(Mockito.anyInt()); _snapshotMgr.persistSnapshotPolicy(volumeMock, TEST_SNAPSHOT_POLICY_SCHEDULE, TEST_SNAPSHOT_POLICY_TIMEZONE, TEST_SNAPSHOT_POLICY_INTERVAL, TEST_SNAPSHOT_POLICY_MAX_SNAPS, - TEST_SNAPSHOT_POLICY_DISPLAY, TEST_SNAPSHOT_POLICY_ACTIVE, mapStringStringMock); + TEST_SNAPSHOT_POLICY_DISPLAY, TEST_SNAPSHOT_POLICY_ACTIVE, mapStringStringMock, null); } } - @Test - public void validatePersistSnapshotPolicyLockAquiredCreateSnapshotPolicy() { + private void testPersistSnapshotPolicyLockAcquired(boolean forUpdate) { try (MockedStatic ignored = Mockito.mockStatic(GlobalLock.class)) { BDDMockito.given(GlobalLock.getInternLock(Mockito.anyString())).willReturn(globalLockMock); Mockito.doReturn(true).when(globalLockMock).lock(Mockito.anyInt()); + List persistedPolicies = new ArrayList<>(); + List updatedPolicies = new ArrayList<>(); + Mockito.when(snapshotPolicyDaoMock.persist(Mockito.any(SnapshotPolicyVO.class))).thenAnswer((Answer) invocation -> { + SnapshotPolicyVO policy = (SnapshotPolicyVO)invocation.getArguments()[0]; + persistedPolicies.add(policy); + return policy; + }); + Mockito.when(snapshotPolicyDaoMock.update(Mockito.anyLong(), Mockito.any(SnapshotPolicyVO.class))).thenAnswer((Answer) invocation -> { + SnapshotPolicyVO policy = (SnapshotPolicyVO)invocation.getArguments()[1]; + updatedPolicies.add(policy); + return true; + }); for (IntervalType intervalType : listIntervalTypes) { - - Mockito.doReturn(null).when(snapshotPolicyDaoMock).findOneByVolumeInterval(anyLong(), Mockito.eq(intervalType)); - Mockito.doReturn(snapshotPolicyVoInstance).when(_snapshotMgr).createSnapshotPolicy(anyLong(), Mockito.anyString(), Mockito.anyString(), Mockito.eq(intervalType), - Mockito.anyInt(), Mockito.anyBoolean()); - + Mockito.doReturn(forUpdate ? snapshotPolicyVoInstance : null).when(snapshotPolicyDaoMock).findOneByVolumeInterval(Mockito.anyLong(), Mockito.eq(intervalType)); SnapshotPolicyVO result = _snapshotMgr.persistSnapshotPolicy(volumeMock, TEST_SNAPSHOT_POLICY_SCHEDULE, TEST_SNAPSHOT_POLICY_TIMEZONE, intervalType, - TEST_SNAPSHOT_POLICY_MAX_SNAPS, TEST_SNAPSHOT_POLICY_DISPLAY, TEST_SNAPSHOT_POLICY_ACTIVE, null); + TEST_SNAPSHOT_POLICY_MAX_SNAPS, TEST_SNAPSHOT_POLICY_DISPLAY, TEST_SNAPSHOT_POLICY_ACTIVE, null, null); - assertSnapshotPolicyResultAgainstPreBuiltInstance(result); + assertSnapshotPolicyResultAgainstPreBuiltInstance(result, (short)intervalType.ordinal()); } - VerificationMode timesVerification = Mockito.times(listIntervalTypes.size()); - Mockito.verify(_snapshotMgr, timesVerification).createSnapshotPolicy(anyLong(), Mockito.anyString(), Mockito.anyString(), any(DateUtil.IntervalType.class), - Mockito.anyInt(), Mockito.anyBoolean()); - Mockito.verify(_snapshotMgr, Mockito.never()).updateSnapshotPolicy(any(SnapshotPolicyVO.class), Mockito.anyString(), Mockito.anyString(), - any(DateUtil.IntervalType.class), Mockito.anyInt(), Mockito.anyBoolean(), Mockito.anyBoolean()); - Mockito.verify(_snapshotMgr, timesVerification).createTagsForSnapshotPolicy(any(), any()); + Assert.assertEquals(forUpdate ? 0 : listIntervalTypes.size(), persistedPolicies.size()); + Assert.assertEquals(forUpdate ? listIntervalTypes.size() : 0, updatedPolicies.size()); + Mockito.verify(taggedResourceServiceMock, Mockito.never()).createTags(Mockito.anyList(), Mockito.any(ResourceTag.ResourceObjectType.class), Mockito.anyMap(), Mockito.anyString()); } } @Test - public void validatePersistSnapshotPolicyLockAquiredUpdateSnapshotPolicy() { - try (MockedStatic ignored = Mockito.mockStatic(GlobalLock.class)) { - - BDDMockito.given(GlobalLock.getInternLock(Mockito.anyString())).willReturn(globalLockMock); - Mockito.doReturn(true).when(globalLockMock).lock(Mockito.anyInt()); - - for (IntervalType intervalType : listIntervalTypes) { - Mockito.doReturn(snapshotPolicyVoInstance).when(snapshotPolicyDaoMock).findOneByVolumeInterval(anyLong(), Mockito.eq(intervalType)); - Mockito.doNothing().when(_snapshotMgr).updateSnapshotPolicy(any(SnapshotPolicyVO.class), Mockito.anyString(), Mockito.anyString(), - any(DateUtil.IntervalType.class), Mockito.anyInt(), Mockito.anyBoolean(), Mockito.anyBoolean()); - - SnapshotPolicyVO result = _snapshotMgr.persistSnapshotPolicy(volumeMock, TEST_SNAPSHOT_POLICY_SCHEDULE, TEST_SNAPSHOT_POLICY_TIMEZONE, intervalType, - TEST_SNAPSHOT_POLICY_MAX_SNAPS, TEST_SNAPSHOT_POLICY_DISPLAY, TEST_SNAPSHOT_POLICY_ACTIVE, null); - - assertSnapshotPolicyResultAgainstPreBuiltInstance(result); - } + public void validatePersistSnapshotPolicyLockAcquiredCreateSnapshotPolicy() { + testPersistSnapshotPolicyLockAcquired(false); + } - VerificationMode timesVerification = Mockito.times(listIntervalTypes.size()); - Mockito.verify(_snapshotMgr, Mockito.never()).createSnapshotPolicy(anyLong(), Mockito.anyString(), Mockito.anyString(), any(DateUtil.IntervalType.class), - Mockito.anyInt(), Mockito.anyBoolean()); - Mockito.verify(_snapshotMgr, timesVerification).updateSnapshotPolicy(any(SnapshotPolicyVO.class), Mockito.anyString(), Mockito.anyString(), - any(DateUtil.IntervalType.class), Mockito.anyInt(), Mockito.anyBoolean(), Mockito.anyBoolean()); - Mockito.verify(_snapshotMgr, timesVerification).createTagsForSnapshotPolicy(any(), any()); - } + @Test + public void validatePersistSnapshotPolicyLockAcquiredUpdateSnapshotPolicy() { + testPersistSnapshotPolicyLockAcquired(true); } private void mockForBackupSnapshotToSecondaryZoneTest(final Boolean configValue, final DataCenter.Type dcType) { diff --git a/server/src/test/java/com/cloud/template/TemplateManagerImplTest.java b/server/src/test/java/com/cloud/template/TemplateManagerImplTest.java index 8331cff02771..43c3b3f25c07 100755 --- a/server/src/test/java/com/cloud/template/TemplateManagerImplTest.java +++ b/server/src/test/java/com/cloud/template/TemplateManagerImplTest.java @@ -19,6 +19,75 @@ package com.cloud.template; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.nullable; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyBoolean; +import static org.mockito.Matchers.anyLong; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import javax.inject.Inject; + +import org.apache.cloudstack.api.command.user.template.CreateTemplateCmd; +import org.apache.cloudstack.api.command.user.template.DeleteTemplateCmd; +import org.apache.cloudstack.api.command.user.userdata.LinkUserDataToTemplateCmd; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; +import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector; +import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotService; +import org.apache.cloudstack.engine.subsystem.api.storage.StorageCacheManager; +import org.apache.cloudstack.engine.subsystem.api.storage.StorageStrategyFactory; +import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory; +import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.framework.messagebus.MessageBus; +import org.apache.cloudstack.snapshot.SnapshotHelper; +import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; +import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; +import org.apache.cloudstack.test.utils.SpringUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.FilterType; +import org.springframework.core.type.classreading.MetadataReader; +import org.springframework.core.type.classreading.MetadataReaderFactory; +import org.springframework.core.type.filter.TypeFilter; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; +import org.springframework.test.context.support.AnnotationConfigContextLoader; + import com.cloud.agent.AgentManager; import com.cloud.api.query.dao.UserVmJoinDao; import com.cloud.configuration.Resource; @@ -66,73 +135,6 @@ import com.cloud.vm.VMInstanceVO; import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.dao.VMInstanceDao; -import org.apache.cloudstack.api.command.user.template.CreateTemplateCmd; -import org.apache.cloudstack.api.command.user.template.DeleteTemplateCmd; -import org.apache.cloudstack.api.command.user.userdata.LinkUserDataToTemplateCmd; -import org.apache.cloudstack.context.CallContext; -import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService; -import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; -import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector; -import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStore; -import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory; -import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotService; -import org.apache.cloudstack.engine.subsystem.api.storage.StorageCacheManager; -import org.apache.cloudstack.engine.subsystem.api.storage.StorageStrategyFactory; -import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory; -import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService; -import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory; -import org.apache.cloudstack.framework.config.dao.ConfigurationDao; -import org.apache.cloudstack.framework.messagebus.MessageBus; -import org.apache.cloudstack.snapshot.SnapshotHelper; -import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; -import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; -import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; -import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; -import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; -import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; -import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; -import org.apache.cloudstack.test.utils.SpringUtils; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Mockito; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.ComponentScan; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.FilterType; -import org.springframework.core.type.classreading.MetadataReader; -import org.springframework.core.type.classreading.MetadataReaderFactory; -import org.springframework.core.type.filter.TypeFilter; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.test.context.support.AnnotationConfigContextLoader; - -import javax.inject.Inject; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.UUID; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; - -import static org.junit.Assert.assertTrue; -import static org.mockito.ArgumentMatchers.nullable; -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anyBoolean; -import static org.mockito.Matchers.anyLong; -import static org.mockito.Mockito.doNothing; -import static org.mockito.Mockito.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(loader = AnnotationConfigContextLoader.class) @@ -477,6 +479,7 @@ public void testCreatePrivateTemplateRecordForRegionStore() throws ResourceAlloc when(mockCreateCmd.getOsTypeId()).thenReturn(1L); when(mockCreateCmd.getEventDescription()).thenReturn("test"); when(mockCreateCmd.getDetails()).thenReturn(null); + when(mockCreateCmd.getZoneId()).thenReturn(null); Account mockTemplateOwner = mock(Account.class); diff --git a/server/src/test/java/org/apache/cloudstack/snapshot/SnapshotHelperTest.java b/server/src/test/java/org/apache/cloudstack/snapshot/SnapshotHelperTest.java index 990e61bdc588..84e7144e5f62 100644 --- a/server/src/test/java/org/apache/cloudstack/snapshot/SnapshotHelperTest.java +++ b/server/src/test/java/org/apache/cloudstack/snapshot/SnapshotHelperTest.java @@ -19,12 +19,14 @@ package org.apache.cloudstack.snapshot; -import com.cloud.hypervisor.Hypervisor; -import com.cloud.hypervisor.Hypervisor.HypervisorType; -import com.cloud.storage.DataStoreRole; -import com.cloud.storage.VolumeVO; -import com.cloud.storage.dao.SnapshotDao; -import com.cloud.utils.exception.CloudRuntimeException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotService; @@ -40,11 +42,12 @@ import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Set; +import com.cloud.hypervisor.Hypervisor; +import com.cloud.hypervisor.Hypervisor.HypervisorType; +import com.cloud.storage.DataStoreRole; +import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.SnapshotDao; +import com.cloud.utils.exception.CloudRuntimeException; @RunWith(MockitoJUnitRunner.class) public class SnapshotHelperTest { @@ -75,6 +78,9 @@ public class SnapshotHelperTest { @Mock SnapshotDao snapshotDaoMock; + @Mock + DataStoreManager dataStoreManager; + @Mock VolumeVO volumeVoMock; @@ -87,6 +93,7 @@ public void init() { snapshotHelperSpy.snapshotFactory = snapshotDataFactoryMock; snapshotHelperSpy.storageStrategyFactory = storageStrategyFactoryMock; snapshotHelperSpy.snapshotDao = snapshotDaoMock; + snapshotHelperSpy.dataStorageManager = dataStoreManager; } @Test @@ -97,13 +104,17 @@ public void validateExpungeTemporarySnapshotNotAKvmSnapshotOnPrimaryStorageDoNot @Test public void validateExpungeTemporarySnapshotKvmSnapshotOnPrimaryStorageExpungesSnapshot() { + DataStore store = Mockito.mock(DataStore.class); + Mockito.when(store.getRole()).thenReturn(DataStoreRole.Image); + Mockito.when(store.getId()).thenReturn(1L); + Mockito.when(snapshotInfoMock.getDataStore()).thenReturn(store); Mockito.doReturn(true).when(snapshotServiceMock).deleteSnapshot(Mockito.any()); - Mockito.doReturn(true).when(snapshotDataStoreDaoMock).expungeReferenceBySnapshotIdAndDataStoreRole(Mockito.anyLong(), Mockito.any()); + Mockito.doReturn(true).when(snapshotDataStoreDaoMock).expungeReferenceBySnapshotIdAndDataStoreRole(Mockito.anyLong(), Mockito.anyLong(), Mockito.any()); snapshotHelperSpy.expungeTemporarySnapshot(true, snapshotInfoMock); Mockito.verify(snapshotServiceMock).deleteSnapshot(Mockito.any()); - Mockito.verify(snapshotDataStoreDaoMock).expungeReferenceBySnapshotIdAndDataStoreRole(Mockito.anyLong(), Mockito.any()); + Mockito.verify(snapshotDataStoreDaoMock).expungeReferenceBySnapshotIdAndDataStoreRole(Mockito.anyLong(), Mockito.anyLong(), Mockito.any()); } @Test @@ -138,20 +149,25 @@ public void validateIsKvmSnapshotOnlyInPrimaryStorageBackupToSecondaryFalse() { @Test public void validateGetSnapshotInfoByIdAndRoleSnapInfoFoundReturnIt() { - Mockito.doReturn(snapshotInfoMock).when(snapshotDataFactoryMock).getSnapshot(Mockito.anyLong(), Mockito.any(DataStoreRole.class)); + Mockito.doReturn(snapshotInfoMock).when(snapshotDataFactoryMock).getSnapshotOnPrimaryStore(Mockito.anyLong()); + Mockito.doReturn(snapshotInfoMock).when(snapshotDataFactoryMock).getSnapshotWithRoleAndZone(Mockito.anyLong(), Mockito.any(DataStoreRole.class), Mockito.anyLong()); dataStoreRoles.forEach(role -> { - SnapshotInfo result = snapshotHelperSpy.getSnapshotInfoByIdAndRole(0, role); + SnapshotInfo result = snapshotHelperSpy.getSnapshotInfoByIdAndRole(0, role, 1L); Assert.assertEquals(snapshotInfoMock, result); }); } - @Test(expected = CloudRuntimeException.class) + @Test public void validateGetSnapshotInfoByIdAndRoleSnapInfoNotFoundThrowCloudRuntimeException() { - Mockito.doReturn(null).when(snapshotDataFactoryMock).getSnapshot(Mockito.anyLong(), Mockito.any(DataStoreRole.class)); + Mockito.doReturn(null).when(snapshotDataFactoryMock).getSnapshotOnPrimaryStore(Mockito.anyLong()); + Mockito.doReturn(null).when(snapshotDataFactoryMock).getSnapshotWithRoleAndZone(Mockito.anyLong(), Mockito.any(DataStoreRole.class), Mockito.anyLong()); dataStoreRoles.forEach(role -> { - snapshotHelperSpy.getSnapshotInfoByIdAndRole(0, role); + try { + snapshotHelperSpy.getSnapshotInfoByIdAndRole(0, role, 1L); + Assert.fail(String.format("Expected a CloudRuntimeException for datastore role: %s", role)); + } catch (CloudRuntimeException ignored) {} }); } @@ -188,7 +204,7 @@ public void validateIsSnapshotBackupableSnapInfoNotNullAndAllRolesAndKvmSnapshot } @Test - public void validateBackupSnapshotToSecondaryStorageIfNotExistsSnapshotIsNotBackupable(){ + public void validateBackupSnapshotToSecondaryStorageIfNotExistsSnapshotIsNotBackupable() { Mockito.doReturn(false).when(snapshotHelperSpy).isSnapshotBackupable(Mockito.any(), Mockito.any(), Mockito.anyBoolean()); SnapshotInfo result = snapshotHelperSpy.backupSnapshotToSecondaryStorageIfNotExists(snapshotInfoMock, DataStoreRole.Image, snapshotInfoMock, true); Assert.assertEquals(snapshotInfoMock, result); @@ -197,7 +213,7 @@ public void validateBackupSnapshotToSecondaryStorageIfNotExistsSnapshotIsNotBack @Test (expected = CloudRuntimeException.class) public void validateBackupSnapshotToSecondaryStorageIfNotExistsGetSnapshotThrowsCloudRuntimeException(){ Mockito.doReturn(true).when(snapshotHelperSpy).isSnapshotBackupable(Mockito.any(), Mockito.any(), Mockito.anyBoolean()); - Mockito.doThrow(CloudRuntimeException.class).when(snapshotHelperSpy).getSnapshotInfoByIdAndRole(Mockito.anyLong(), Mockito.any()); + Mockito.when(snapshotDataFactoryMock.getSnapshotOnPrimaryStore(Mockito.anyLong())).thenReturn(null); snapshotHelperSpy.backupSnapshotToSecondaryStorageIfNotExists(snapshotInfoMock, DataStoreRole.Image, snapshotInfoMock, true); } @@ -205,7 +221,9 @@ public void validateBackupSnapshotToSecondaryStorageIfNotExistsGetSnapshotThrows @Test public void validateBackupSnapshotToSecondaryStorageIfNotExistsReturnSnapshotInfo(){ Mockito.doReturn(true).when(snapshotHelperSpy).isSnapshotBackupable(Mockito.any(), Mockito.any(), Mockito.anyBoolean()); - Mockito.doReturn(snapshotInfoMock, snapshotInfoMock2).when(snapshotHelperSpy).getSnapshotInfoByIdAndRole(Mockito.anyLong(), Mockito.any()); + Mockito.when(snapshotInfoMock.getDataStore()).thenReturn(Mockito.mock(DataStore.class)); + Mockito.when(snapshotDataFactoryMock.getSnapshotOnPrimaryStore(Mockito.anyLong())).thenReturn(snapshotInfoMock); + Mockito.when(snapshotDataFactoryMock.getSnapshotWithRoleAndZone(Mockito.anyLong(), Mockito.any(DataStoreRole.class), Mockito.anyLong())).thenReturn(snapshotInfoMock2); Mockito.doReturn(snapshotStrategyMock).when(storageStrategyFactoryMock).getSnapshotStrategy(Mockito.any(), Mockito.any()); Mockito.doReturn(null).when(snapshotStrategyMock).backupSnapshot(Mockito.any()); diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java index e47e5a67b4a7..4883bb77dca1 100644 --- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java +++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java @@ -49,6 +49,8 @@ import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.stream.Collectors; +import java.util.stream.Stream; import javax.naming.ConfigurationException; @@ -60,6 +62,8 @@ import org.apache.cloudstack.storage.command.DownloadCommand; import org.apache.cloudstack.storage.command.DownloadProgressCommand; import org.apache.cloudstack.storage.command.MoveVolumeCommand; +import org.apache.cloudstack.storage.command.QuerySnapshotZoneCopyAnswer; +import org.apache.cloudstack.storage.command.QuerySnapshotZoneCopyCommand; import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand; import org.apache.cloudstack.storage.command.UploadStatusAnswer; import org.apache.cloudstack.storage.command.UploadStatusAnswer.UploadStatus; @@ -318,6 +322,8 @@ public Answer executeRequest(Command cmd) { return execute((CreateDatadiskTemplateCommand)cmd); } else if (cmd instanceof MoveVolumeCommand) { return execute((MoveVolumeCommand)cmd); + } else if (cmd instanceof QuerySnapshotZoneCopyCommand) { + return execute((QuerySnapshotZoneCopyCommand)cmd); } else { return Answer.createUnsupportedCommandAnswer(cmd); } @@ -3588,4 +3594,32 @@ private TemplateOrVolumePostUploadCommand getTemplateOrVolumePostUploadCmd(Strin return cmd; } + protected Answer execute(QuerySnapshotZoneCopyCommand cmd) { + SnapshotObjectTO snapshot = cmd.getSnapshot(); + String parentPath = getRootDir(snapshot.getDataStore().getUrl(), _nfsVersion); + String path = snapshot.getPath(); + File snapFile = new File(parentPath + File.separator + path); + if (snapFile.exists() && !snapFile.isDirectory()) { + return new QuerySnapshotZoneCopyAnswer(cmd, List.of(path)); + } + int index = path.lastIndexOf(File.separator); + String snapDir = path.substring(0, index); + List files = new ArrayList<>(); + try (Stream stream = Files.list(Paths.get(parentPath + File.separator + snapDir))) { + List fileNames = stream + .filter(file -> !Files.isDirectory(file)) + .map(Path::getFileName) + .map(Path::toString) + .collect(Collectors.toList()); + for (String file : fileNames) { + file = snapDir + "/" + file; + s_logger.debug(String.format("Found snapshot file %s", file)); + files.add(file); + } + } catch (IOException ioe) { + s_logger.error("Error preparing file list for snapshot copy", ioe); + } + return new QuerySnapshotZoneCopyAnswer(cmd, files); + } + } diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java index f647b497f586..0396f96f094d 100644 --- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java +++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java @@ -16,12 +16,18 @@ // under the License. package org.apache.cloudstack.storage.template; +import static com.cloud.utils.NumbersUtil.toHumanReadableSize; + import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.net.URI; import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; import java.security.NoSuchAlgorithmException; import java.text.SimpleDateFormat; import java.util.ArrayList; @@ -37,55 +43,54 @@ import javax.naming.ConfigurationException; -import com.cloud.agent.api.to.OVFInformationTO; -import com.cloud.storage.template.Processor; -import com.cloud.storage.template.S3TemplateDownloader; -import com.cloud.storage.template.TemplateDownloader; -import com.cloud.storage.template.TemplateLocation; -import com.cloud.storage.template.MetalinkTemplateDownloader; -import com.cloud.storage.template.HttpTemplateDownloader; -import com.cloud.storage.template.LocalTemplateDownloader; -import com.cloud.storage.template.ScpTemplateDownloader; -import com.cloud.storage.template.TemplateProp; -import com.cloud.storage.template.OVAProcessor; -import com.cloud.storage.template.IsoProcessor; -import com.cloud.storage.template.QCOW2Processor; -import com.cloud.storage.template.VmdkProcessor; -import com.cloud.storage.template.RawImageProcessor; -import com.cloud.storage.template.TARProcessor; -import com.cloud.storage.template.VhdProcessor; -import com.cloud.storage.template.TemplateConstants; +import org.apache.cloudstack.storage.NfsMountManagerImpl.PathParser; import org.apache.cloudstack.storage.command.DownloadCommand; import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType; import org.apache.cloudstack.storage.command.DownloadProgressCommand; import org.apache.cloudstack.storage.command.DownloadProgressCommand.RequestType; -import org.apache.cloudstack.storage.NfsMountManagerImpl.PathParser; import org.apache.cloudstack.storage.resource.NfsSecondaryStorageResource; import org.apache.cloudstack.storage.resource.SecondaryStorageResource; +import org.apache.cloudstack.utils.security.ChecksumValue; +import org.apache.cloudstack.utils.security.DigestHelper; +import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; import com.cloud.agent.api.storage.DownloadAnswer; -import com.cloud.utils.net.Proxy; import com.cloud.agent.api.to.DataStoreTO; import com.cloud.agent.api.to.NfsTO; +import com.cloud.agent.api.to.OVFInformationTO; import com.cloud.agent.api.to.S3TO; import com.cloud.exception.InternalErrorException; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.StorageLayer; import com.cloud.storage.VMTemplateStorageResourceAssoc; +import com.cloud.storage.template.SimpleHttpMultiFileDownloader; +import com.cloud.storage.template.HttpTemplateDownloader; +import com.cloud.storage.template.IsoProcessor; +import com.cloud.storage.template.LocalTemplateDownloader; +import com.cloud.storage.template.MetalinkTemplateDownloader; +import com.cloud.storage.template.OVAProcessor; +import com.cloud.storage.template.Processor; import com.cloud.storage.template.Processor.FormatInfo; +import com.cloud.storage.template.QCOW2Processor; +import com.cloud.storage.template.RawImageProcessor; +import com.cloud.storage.template.S3TemplateDownloader; +import com.cloud.storage.template.ScpTemplateDownloader; +import com.cloud.storage.template.TARProcessor; +import com.cloud.storage.template.TemplateConstants; +import com.cloud.storage.template.TemplateDownloader; import com.cloud.storage.template.TemplateDownloader.DownloadCompleteCallback; import com.cloud.storage.template.TemplateDownloader.Status; +import com.cloud.storage.template.TemplateLocation; +import com.cloud.storage.template.TemplateProp; +import com.cloud.storage.template.VhdProcessor; +import com.cloud.storage.template.VmdkProcessor; import com.cloud.utils.NumbersUtil; import com.cloud.utils.component.ManagerBase; import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.net.Proxy; import com.cloud.utils.script.Script; import com.cloud.utils.storage.QCOW2Utils; -import org.apache.cloudstack.utils.security.ChecksumValue; -import org.apache.cloudstack.utils.security.DigestHelper; -import org.apache.commons.lang3.StringUtils; - -import static com.cloud.utils.NumbersUtil.toHumanReadableSize; public class DownloadManagerImpl extends ManagerBase implements DownloadManager { private String _name; @@ -186,17 +191,31 @@ public String getInstallPathPrefix() { return installPathPrefix; } + private void cleanupFileWithDirectory(String path, boolean deleteDir) { + if (StringUtils.isEmpty(path)) { + return; + } + LOGGER.debug(String.format("Cleaning-up temporary download file %s", path)); + File f = new File(path); + File dir = f.getParentFile(); + f.delete(); + if (deleteDir && dir != null) { + LOGGER.debug(String.format("Deleting directory %s, if empty, as part of cleanup", dir.getAbsolutePath())); + dir.delete(); + } + } + public void cleanup() { if (td != null) { - String dnldPath = td.getDownloadLocalPath(); - if (dnldPath != null) { - File f = new File(dnldPath); - File dir = f.getParentFile(); - f.delete(); - if (dir != null) { - dir.delete(); + if (td instanceof SimpleHttpMultiFileDownloader) { + SimpleHttpMultiFileDownloader httpMultiFileDownloader = (SimpleHttpMultiFileDownloader)td; + List files = new ArrayList<>(httpMultiFileDownloader.getDownloadedFilesMap().values()); + for (int i = 0; i < files.size(); ++i) { + cleanupFileWithDirectory(files.get(i), i == files.size() - 1); } + return; } + cleanupFileWithDirectory(td.getDownloadLocalPath(), true); } } @@ -304,8 +323,7 @@ public void setDownloadStatus(String jobId, Status status) { td.setStatus(Status.POST_DOWNLOAD_FINISHED); td.setDownloadError("Install completed successfully at " + new SimpleDateFormat().format(new Date())); } - } - else { + } else { // For other TemplateDownloaders where files are locally available, // we run the postLocalDownload() method. td.setDownloadError("Download success, starting install "); @@ -366,6 +384,98 @@ private String postRemoteDownload(String jobId) { return result; } + protected String getSnapshotInstallNameFromDownloadUrl(String url) { + URI uri; + try { + uri = new URI(url); + } catch (URISyntaxException ignored) { + return null; + } + String name = uri.getPath(); + if (StringUtils.isEmpty(name) || !name.contains("/")) { + return null; + } + String[] items = uri.getPath().split("/"); + name = items[items.length - 1]; + if (items.length < 2) { + return name; + } + String parentDir = items[items.length - 2]; + if (!parentDir.matches("\\d+") && name.startsWith(parentDir)) { + return parentDir + File.separator + name; + } + return name; + } + + private String postLocalSnapshotSingleFileDownload(DownloadJob job, HttpTemplateDownloader td) { + String name = getSnapshotInstallNameFromDownloadUrl(td.getDownloadUrl()); + final String downloadedFile = td.getDownloadLocalPath(); + final String resourcePath = job.getInstallPathPrefix(); + final String relativeResourcePath = job.getTmpltPath(); + if (StringUtils.isEmpty(name)) { + name = UUID.randomUUID().toString(); + LOGGER.warn(String.format("Unable to retrieve install filename for snapshot download %s, using a random UUID", downloadedFile)); + } + Path srcPath = Paths.get(downloadedFile); + Path destPath = Paths.get(resourcePath + File.separator + name); + try { + LOGGER.debug(String.format("Trying to create missing directories (if any) to move snapshot %s.", destPath)); + Files.createDirectories(destPath.getParent()); + LOGGER.debug(String.format("Trying to move downloaded snapshot [%s] to [%s].", srcPath, destPath)); + Files.move(srcPath, destPath, StandardCopyOption.REPLACE_EXISTING); + } catch (IOException e) { + LOGGER.warn(String.format("Something is wrong while processing post snapshot download %s", resourcePath), e); + return "Unable process post snapshot download due to " + e.getMessage(); + } + String installedPath = relativeResourcePath + File.separator + name; + job.setTmpltPath(installedPath); + job.setTemplatePhysicalSize(td.getDownloadedBytes()); + return null; + } + + private String postLocalSnapshotMultiFileDownload(DownloadJob job, SimpleHttpMultiFileDownloader td) { + Map downloads = td.getDownloadedFilesMap(); + String installDir = null; + try { + for (Map.Entry entry : downloads.entrySet()) { + final String url = entry.getKey(); + final String downloadedFile = entry.getValue(); + final String name = url.substring(url.lastIndexOf("/")); + if (StringUtils.isEmpty(installDir)) { + installDir = url.substring(0, url.lastIndexOf("/")); + installDir = installDir.substring(installDir.lastIndexOf("/")); + job.setTmpltPath(job.getTmpltPath() + installDir); + installDir = job.getInstallPathPrefix() + installDir; + Path installPath = Paths.get(installDir); + LOGGER.debug(String.format("Trying to create missing directories (if any) to move snapshot files at %s.", installDir)); + Files.createDirectories(installPath); + } + final String filePath = installDir + name; + if (name.endsWith(".ovf")) { + job.setTmpltPath(job.getTmpltPath() + name.replace(".ovf", "")); + } + Path srcPath = Paths.get(downloadedFile); + Path destPath = Paths.get(filePath); + LOGGER.debug(String.format("Trying to move downloaded snapshot file [%s] to [%s].", srcPath, destPath)); + Files.move(srcPath, destPath, StandardCopyOption.REPLACE_EXISTING); + } + job.setTemplatePhysicalSize(td.getDownloadedBytes()); + } catch (IOException e) { + LOGGER.warn(String.format("Something is wrong while processing post snapshot download %s", job.getTmpltPath()), e); + return "Unable process post snapshot download due to " + e.getMessage(); + } + return null; + } + + private String postLocalSnapshotDownload(DownloadJob job, TemplateDownloader td) { + if (td instanceof HttpTemplateDownloader) { + return postLocalSnapshotSingleFileDownload(job, (HttpTemplateDownloader)td); + } else if(td instanceof SimpleHttpMultiFileDownloader) { + return postLocalSnapshotMultiFileDownload(job, (SimpleHttpMultiFileDownloader)td); + } + return null; + } + /** * Post local download activity (install and cleanup). Executed in context of * downloader thread @@ -376,13 +486,12 @@ private String postRemoteDownload(String jobId) { private String postLocalDownload(String jobId) { DownloadJob dnld = jobs.get(jobId); TemplateDownloader td = dnld.getTemplateDownloader(); - String resourcePath = dnld.getInstallPathPrefix(); // path with mount - // directory - String finalResourcePath = dnld.getTmpltPath(); // template download - // path on secondary - // storage ResourceType resourceType = dnld.getResourceType(); - + if (ResourceType.SNAPSHOT.equals(resourceType)) { + return postLocalSnapshotDownload(dnld, td); + } + String resourcePath = dnld.getInstallPathPrefix(); // path with mount directory + String finalResourcePath = dnld.getTmpltPath(); // template download path on secondary storage File originalTemplate = new File(td.getDownloadLocalPath()); if(StringUtils.isBlank(dnld.getChecksum())) { if (LOGGER.isInfoEnabled()) { @@ -409,7 +518,7 @@ private String postLocalDownload(String jobId) { File downloadedTemplate = new File(resourcePath + "/" + templateFilename); _storage.setWorldReadableAndWriteable(downloadedTemplate); - setPermissionsForTheDownloadedTemplate(dnld, resourcePath, resourceType); + setPermissionsForTheDownloadedTemplate(resourcePath, resourceType); TemplateLocation loc = new TemplateLocation(_storage, resourcePath); try { @@ -468,7 +577,10 @@ private String makeTemplatename(String jobId, String extension) { return templateName; } - private void setPermissionsForTheDownloadedTemplate(DownloadJob dnld, String resourcePath, ResourceType resourceType) { + private void setPermissionsForTheDownloadedTemplate(String resourcePath, ResourceType resourceType) { + if (ResourceType.SNAPSHOT.equals(resourceType)) { + return; + } // Set permissions for template/volume.properties String propertiesFile = resourcePath; if (resourceType == ResourceType.TEMPLATE) { @@ -578,6 +690,32 @@ public String downloadS3Template(S3TO s3, long id, String url, String name, Imag return jobId; } + private String createTempDirAndPropertiesFile(ResourceType resourceType, String tmpDir) throws IOException { + if (!_storage.mkdirs(tmpDir)) { + LOGGER.warn("Unable to create " + tmpDir); + return "Unable to create " + tmpDir; + } + if (ResourceType.SNAPSHOT.equals(resourceType)) { + return null; + } + // TO DO - define constant for volume properties. + File file = + ResourceType.TEMPLATE == resourceType ? + _storage.getFile(tmpDir + File.separator + TemplateLocation.Filename) : + _storage.getFile(tmpDir + File.separator + "volume.properties"); + if (file.exists()) { + if(! file.delete()) { + LOGGER.warn("Deletion of file '" + file.getAbsolutePath() + "' failed."); + } + } + + if (!file.createNewFile()) { + LOGGER.warn("Unable to create new file: " + file.getAbsolutePath()); + return "Unable to create new file: " + file.getAbsolutePath(); + } + return null; + } + @Override public String downloadPublicTemplate(long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum, String installPathPrefix, String templatePath, String user, String password, long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType) { @@ -586,63 +724,58 @@ public String downloadPublicTemplate(long id, String url, String name, ImageForm String tmpDir = installPathPrefix; try { - - if (!_storage.mkdirs(tmpDir)) { - LOGGER.warn("Unable to create " + tmpDir); - return "Unable to create " + tmpDir; + String filesError = createTempDirAndPropertiesFile(resourceType, tmpDir); + if (StringUtils.isNotEmpty(filesError)) { + return filesError; } - // TO DO - define constant for volume properties. - File file = - ResourceType.TEMPLATE == resourceType ? _storage.getFile(tmpDir + File.separator + TemplateLocation.Filename) : _storage.getFile(tmpDir + File.separator + - "volume.properties"); - if (file.exists()) { - if(! file.delete()) { - LOGGER.warn("Deletion of file '" + file.getAbsolutePath() + "' failed."); - } - } - - if (!file.createNewFile()) { - LOGGER.warn("Unable to create new file: " + file.getAbsolutePath()); - return "Unable to create new file: " + file.getAbsolutePath(); - } - URI uri; - try { - uri = new URI(url); - } catch (URISyntaxException e) { - throw new CloudRuntimeException("URI is incorrect: " + url); - } - TemplateDownloader td; - if ((uri != null) && (uri.getScheme() != null)) { - if (uri.getPath().endsWith(".metalink")) { - td = new MetalinkTemplateDownloader(_storage, url, tmpDir, new Completion(jobId), maxTemplateSizeInBytes); - } else if (uri.getScheme().equalsIgnoreCase("http") || uri.getScheme().equalsIgnoreCase("https")) { - td = new HttpTemplateDownloader(_storage, url, tmpDir, new Completion(jobId), maxTemplateSizeInBytes, user, password, proxy, resourceType); - } else if (uri.getScheme().equalsIgnoreCase("file")) { - td = new LocalTemplateDownloader(_storage, url, tmpDir, maxTemplateSizeInBytes, new Completion(jobId)); - } else if (uri.getScheme().equalsIgnoreCase("scp")) { - td = new ScpTemplateDownloader(_storage, url, tmpDir, maxTemplateSizeInBytes, new Completion(jobId)); - } else if (uri.getScheme().equalsIgnoreCase("nfs") || uri.getScheme().equalsIgnoreCase("cifs")) { - td = null; - // TODO: implement this. - throw new CloudRuntimeException("Scheme is not supported " + url); - } else { - throw new CloudRuntimeException("Scheme is not supported " + url); - } + URI uri; + String checkUrl = url; + if (ResourceType.SNAPSHOT.equals(resourceType) && url.contains("\n")) { + checkUrl = url.substring(0, url.indexOf("\n") - 1); + } + try { + uri = new URI(checkUrl); + } catch (URISyntaxException e) { + throw new CloudRuntimeException("URI is incorrect: " + url); + } + TemplateDownloader td; + if (ResourceType.SNAPSHOT.equals(resourceType) && url.contains("\n") && + ("http".equalsIgnoreCase(uri.getScheme()) || "https".equalsIgnoreCase(uri.getScheme()))) { + String[] urls = url.split("\n"); + td = new SimpleHttpMultiFileDownloader(_storage, urls, tmpDir, new Completion(jobId), maxTemplateSizeInBytes, resourceType); + } else { + if ((uri != null) && (uri.getScheme() != null)) { + if (uri.getPath().endsWith(".metalink")) { + td = new MetalinkTemplateDownloader(_storage, url, tmpDir, new Completion(jobId), maxTemplateSizeInBytes); + } else if (uri.getScheme().equalsIgnoreCase("http") || uri.getScheme().equalsIgnoreCase("https")) { + td = new HttpTemplateDownloader(_storage, url, tmpDir, new Completion(jobId), maxTemplateSizeInBytes, user, password, proxy, resourceType); + } else if (uri.getScheme().equalsIgnoreCase("file")) { + td = new LocalTemplateDownloader(_storage, url, tmpDir, maxTemplateSizeInBytes, new Completion(jobId)); + } else if (uri.getScheme().equalsIgnoreCase("scp")) { + td = new ScpTemplateDownloader(_storage, url, tmpDir, maxTemplateSizeInBytes, new Completion(jobId)); + } else if (uri.getScheme().equalsIgnoreCase("nfs") || uri.getScheme().equalsIgnoreCase("cifs")) { + td = null; + // TODO: implement this. + throw new CloudRuntimeException("Scheme is not supported " + url); } else { - throw new CloudRuntimeException("Unable to download from URL: " + url); + throw new CloudRuntimeException("Scheme is not supported " + url); } - // NOTE the difference between installPathPrefix and templatePath - // here. instalPathPrefix is the absolute path for template - // including mount directory - // on ssvm, while templatePath is the final relative path on - // secondary storage. - DownloadJob dj = new DownloadJob(td, jobId, id, name, format, hvm, accountId, descr, cksum, installPathPrefix, resourceType); - dj.setTmpltPath(templatePath); - jobs.put(jobId, dj); - threadPool.execute(td); - - return jobId; + } else { + throw new CloudRuntimeException("Unable to download from URL: " + url); + } + } + // NOTE the difference between installPathPrefix and templatePath + // here. instalPathPrefix is the absolute path for template + // including mount directory + // on ssvm, while templatePath is the final relative path on + // secondary storage. + DownloadJob dj = new DownloadJob(td, jobId, id, name, format, hvm, accountId, descr, cksum, installPathPrefix, resourceType); + dj.setTmpltPath(templatePath); + jobs.put(jobId, dj); + threadPool.execute(td); + + return jobId; } catch (IOException e) { LOGGER.warn("Unable to download to " + tmpDir, e); return null; diff --git a/services/secondary-storage/server/src/test/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResourceTest.java b/services/secondary-storage/server/src/test/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResourceTest.java index 2aac2766cf7c..cd6444a8b4b9 100644 --- a/services/secondary-storage/server/src/test/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResourceTest.java +++ b/services/secondary-storage/server/src/test/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResourceTest.java @@ -23,17 +23,26 @@ import static org.mockito.Mockito.spy; import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.stream.Stream; import org.apache.cloudstack.storage.command.DeleteCommand; +import org.apache.cloudstack.storage.command.QuerySnapshotZoneCopyAnswer; +import org.apache.cloudstack.storage.command.QuerySnapshotZoneCopyCommand; +import org.apache.cloudstack.storage.to.SnapshotObjectTO; import org.apache.cloudstack.storage.to.TemplateObjectTO; import org.apache.log4j.Level; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.MockedStatic; import org.mockito.Mockito; import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; +import com.cloud.agent.api.to.DataStoreTO; import com.cloud.test.TestAppender; @RunWith(MockitoJUnitRunner.class) @@ -104,4 +113,32 @@ public void testGetSnapshotFilepathForDelete() { "/snapshots/2/10", "somename"); } + + @Test + public void testExecuteQuerySnapshotZoneCopyCommand() { + final String dir = "/snapshots/2/10/abc"; + final String fileName = "abc"; + DataStoreTO store = Mockito.mock(DataStoreTO.class); + SnapshotObjectTO object = Mockito.mock(SnapshotObjectTO.class); + Mockito.when(object.getDataStore()).thenReturn(store); + Mockito.when(object.getPath()).thenReturn(dir + File.separator + fileName); + QuerySnapshotZoneCopyCommand cmd = Mockito.mock(QuerySnapshotZoneCopyCommand.class); + Mockito.when(cmd.getSnapshot()).thenReturn(object); + Path p1 = Mockito.mock(Path.class); + Mockito.when(p1.getFileName()).thenReturn(p1); + Mockito.when(p1.toString()).thenReturn(fileName + ".vmdk"); + Path p2 = Mockito.mock(Path.class); + Mockito.when(p2.getFileName()).thenReturn(p2); + Mockito.when(p2.toString()).thenReturn(fileName + ".ovf"); + Stream paths = Stream.of(p1, p2); + try (MockedStatic files = Mockito.mockStatic(Files.class)) { + files.when(() -> Files.list(Mockito.any(Path.class))).thenReturn(paths); + files.when(() -> Files.isDirectory(Mockito.any(Path.class))).thenReturn(false); + QuerySnapshotZoneCopyAnswer answer = (QuerySnapshotZoneCopyAnswer)(resource.execute(cmd)); + List result = answer.getFiles(); + Assert.assertEquals(2, result.size()); + Assert.assertEquals(dir + File.separator + fileName + ".vmdk", result.get(0)); + Assert.assertEquals(dir + File.separator + fileName + ".ovf", result.get(1)); + } + } } diff --git a/services/secondary-storage/server/src/test/java/org/apache/cloudstack/storage/template/DownloadManagerImplTest.java b/services/secondary-storage/server/src/test/java/org/apache/cloudstack/storage/template/DownloadManagerImplTest.java new file mode 100644 index 000000000000..5fbf17ec1d12 --- /dev/null +++ b/services/secondary-storage/server/src/test/java/org/apache/cloudstack/storage/template/DownloadManagerImplTest.java @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.template; + +import java.util.Map; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.junit.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +public class DownloadManagerImplTest { + + @InjectMocks + DownloadManagerImpl downloadManager = new DownloadManagerImpl(); + + @Test + public void testGetSnapshotInstallNameFromDownloadUrl() { + Map urlNames = Map.of( + "http://HOST/copy/SecStorage/e7d75b93-08f3-3488-8089-632c5c3854bf/snapshots/2/8/8d4cd8d8-c66f-4cbe-88ce-0bf99e26fe79.vhd", "8d4cd8d8-c66f-4cbe-88ce-0bf99e26fe79.vhd", + "http://HOST/copy/SecStorage/24492d16-66a6-34df-84ea-cc335e7d5b4a/snapshots/2/6/a84ee92d-43cf-4151-908d-1e8ea6c43d35", "a84ee92d-43cf-4151-908d-1e8ea6c43d35", + "http://HOST/copy/SecStorage/0e3ec9a5-e23d-3edc-bc0f-ce6e641e12c3/snapshots/2/28/ce0e1e42-9268-414c-a874-1802d2d7b429/ce0e1e42-9268-414c-a874-1802d2d7b429.vmdk", "ce0e1e42-9268-414c-a874-1802d2d7b429/ce0e1e42-9268-414c-a874-1802d2d7b429.vmdk" + ); + for (Map.Entry entry: urlNames.entrySet()) { + String url = entry.getKey(); + String filename = entry.getValue(); + String name = downloadManager.getSnapshotInstallNameFromDownloadUrl(url); + Assert.assertEquals(filename, name); + } + } +} diff --git a/test/integration/component/test_snapshot_copy.py b/test/integration/component/test_snapshot_copy.py new file mode 100644 index 000000000000..7b9531a3433b --- /dev/null +++ b/test/integration/component/test_snapshot_copy.py @@ -0,0 +1,351 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" BVT tests for volume snapshot copy functionality +""" +# Import Local Modules +from marvin.cloudstackTestCase import cloudstackTestCase +from marvin.cloudstackAPI import (createSnapshot, + deleteSnapshot, + copySnapshot, + createVolume, + createTemplate, + listOsTypes) +from marvin.lib.utils import (cleanup_resources, + random_gen) +from marvin.lib.base import (Account, + Zone, + ServiceOffering, + DiskOffering, + VirtualMachine, + Volume, + Snapshot, + Template) +from marvin.lib.common import (get_domain, + get_zone, + get_template) +from marvin.lib.decoratorGenerators import skipTestIf +from marvin.codes import FAILED, PASS +from nose.plugins.attrib import attr +import logging +# Import System modules +import math + + +_multiprocess_shared_ = True + + +class TestSnapshotCopy(cloudstackTestCase): + + @classmethod + def setUpClass(cls): + testClient = super(TestSnapshotCopy, cls).getClsTestClient() + cls.apiclient = testClient.getApiClient() + cls.services = testClient.getParsedTestDataConfig() + + # Get Zone, Domain and templates + cls.domain = get_domain(cls.apiclient) + cls.zone = get_zone(cls.apiclient, testClient.getZoneForTests()) + cls.services['mode'] = cls.zone.networktype + + cls._cleanup = [] + cls.logger = logging.getLogger('TestSnapshotCopy') + cls.testsNotSupported = False + cls.zones = Zone.list(cls.apiclient) + enabled_core_zones = [] + if not isinstance(cls.zones, list): + cls.testsNotSupported = True + elif len(cls.zones) < 2: + cls.testsNotSupported = True + else: + for z in cls.zones: + if z.type == 'Core' and z.allocationstate == 'Enabled': + enabled_core_zones.append(z) + if len(enabled_core_zones) < 2: + cls.testsNotSupported = True + + if cls.testsNotSupported == True: + self.logger.info("Unsupported") + return + + cls.additional_zone = None + for z in enabled_core_zones: + if z.id != cls.zone.id: + cls.additional_zone = z + + template = get_template( + cls.apiclient, + cls.zone.id, + cls.services["ostype"]) + if template == FAILED: + assert False, "get_template() failed to return template with description %s" % cls.services["ostype"] + + # Set Zones and disk offerings + cls.services["small"]["zoneid"] = cls.zone.id + cls.services["small"]["template"] = template.id + cls.services["iso"]["zoneid"] = cls.zone.id + + cls.account = Account.create( + cls.apiclient, + cls.services["account"], + domainid=cls.domain.id) + cls._cleanup.append(cls.account) + + compute_offering_service = cls.services["service_offerings"]["tiny"].copy() + cls.service_offering = ServiceOffering.create( + cls.apiclient, + compute_offering_service) + cls._cleanup.append(cls.service_offering) + cls.services["virtual_machine"]["zoneid"] = cls.zone.id + cls.services["virtual_machine"]["template"] = template.id + cls.virtual_machine = VirtualMachine.create( + cls.apiclient, + cls.services["virtual_machine"], + accountid=cls.account.name, + domainid=cls.account.domainid, + serviceofferingid=cls.service_offering.id, + mode=cls.services["mode"] + ) + cls._cleanup.append(cls.virtual_machine) + cls.volume = Volume.list( + cls.apiclient, + virtualmachineid=cls.virtual_machine.id, + type='ROOT', + listall=True + )[0] + + @classmethod + def tearDownClass(cls): + super(TestSnapshotCopy, cls).tearDownClass() + + def setUp(self): + self.apiclient = self.testClient.getApiClient() + self.userapiclient = self.testClient.getUserApiClient( + UserName=self.account.name, + DomainName=self.account.domain + ) + self.dbclient = self.testClient.getDbConnection() + self.snapshot_id = None + self.cleanup = [] + + def tearDown(self): + super(TestSnapshotCopy, self).tearDown() + + def create_snapshot(self, apiclient, zoneids): + cmd = createSnapshot.createSnapshotCmd() + cmd.volumeid = self.volume.id + cmd.account = self.account.name + cmd.domainid = self.account.domainid + if zoneids: + cmd.zoneids = zoneids + snapshot = Snapshot(apiclient.createSnapshot(cmd).__dict__) + self.cleanup.append(snapshot) + return snapshot + + def delete_snapshot(self, apiclient, snapshot_id, zone_id=None): + cmd = deleteSnapshot.deleteSnapshotCmd() + cmd.id = snapshot_id + if zone_id: + cmd.zoneid = zone_id + apiclient.deleteSnapshot(cmd) + + def copy_snapshot(self, apiclient, snapshot_id, zone_ids, source_zone_id=None): + cmd = copySnapshot.copySnapshotCmd() + cmd.id = snapshot_id + cmd.destzoneids = zone_ids + if source_zone_id: + cmd.sourcezoneid = source_zone_id + return apiclient.copySnapshot(cmd) + + def create_snapshot_volume(self, apiclient, snapshot_id, zone_id=None, disk_offering_id=None): + cmd = createVolume.createVolumeCmd() + cmd.name = "-".join(["VolumeFromSnap", random_gen()]) + cmd.snapshotid = snapshot_id + if zone_id: + cmd.zoneid = zone_id + if disk_offering_id: + cmd.diskofferingid = disk_offering_id + volume_from_snapshot = Volume(apiclient.createVolume(cmd).__dict__) + self.cleanup.append(volume_from_snapshot) + return volume_from_snapshot + + def create_snapshot_template(self, apiclient, services, snapshot_id, zone_id): + cmd = createTemplate.createTemplateCmd() + cmd.displaytext = "TemplateFromSnap" + name = "-".join([cmd.displaytext, random_gen()]) + cmd.name = name + if "ostypeid" in services: + cmd.ostypeid = services["ostypeid"] + elif "ostype" in services: + # Find OSTypeId from Os type + sub_cmd = listOsTypes.listOsTypesCmd() + sub_cmd.description = services["ostype"] + ostypes = apiclient.listOsTypes(sub_cmd) + + if not isinstance(ostypes, list): + self.fail("Unable to find Ostype id with desc: %s" % + services["ostype"]) + cmd.ostypeid = ostypes[0].id + else: + self.fail("Unable to find Ostype is required for creating template") + + cmd.isfeatured = True + cmd.ispublic = True + cmd.isextractable = False + + cmd.snapshotid = snapshot_id + cmd.zoneid = zone_id + apiclient.createTemplate(cmd) + templates = Template.list(apiclient, name=name, templatefilter="self") + if not isinstance(templates, list) and len(templates) < 0: + self.fail("Unable to find created template with name %s" % name) + template = Template(templates[0].__dict__) + self.cleanup.append(template) + return template + + def verify_snapshot_copies(self, snapshot_id, zone_ids): + snapshot_entries = Snapshot.list(self.userapiclient, id=snapshot_id, showunique=False, locationtype="Secondary") + if not isinstance(snapshot_entries, list): + self.fail("Unable to list snapshot for multiple zones") + elif len(snapshot_entries) != len(zone_ids): + self.fail("Undesired list snapshot size for multiple zones") + for zone_id in zone_ids: + zone_found = False + for entry in snapshot_entries: + if entry.zoneid == zone_id: + zone_found = True + break + if zone_found == False: + self.fail("Unable to find snapshot entry for the zone ID: %s" % zone_id) + + @skipTestIf("testsNotSupported") + @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") + def test_01_take_snapshot_multi_zone(self): + """Test to take volume snapshot in multiple zones + """ + # Validate the following: + # 1. Take snapshot in multiple zone + # 2. Verify + + snapshot = self.create_snapshot(self.userapiclient, [str(self.additional_zone.id)]) + self.snapshot_id = snapshot.id + self.verify_snapshot_copies(self.snapshot_id, [self.zone.id, self.additional_zone.id]) + return + + @skipTestIf("testsNotSupported") + @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") + def test_02_copy_snapshot_multi_zone(self): + """Test to take volume snapshot in a zone and then copy + """ + # Validate the following: + # 1. Take snapshot in the native zone + # 2. Copy snapshot in the additional zone + # 3. Verify + + snapshot = self.create_snapshot(self.userapiclient, None) + self.snapshot_id = snapshot.id + self.copy_snapshot(self.userapiclient, self.snapshot_id, [str(self.additional_zone.id)], self.zone.id) + self.verify_snapshot_copies(self.snapshot_id, [self.zone.id, self.additional_zone.id]) + return + + @skipTestIf("testsNotSupported") + @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") + def test_03_take_snapshot_multi_zone_delete_single_zone(self): + """Test to take volume snapshot in multiple zones and delete from one zone + """ + # Validate the following: + # 1. Take snapshot in multiple zone + # 2. Verify + # 3. Delete from one zone + # 4. Verify + + snapshot = self.create_snapshot(self.userapiclient, [str(self.additional_zone.id)]) + self.snapshot_id = snapshot.id + self.verify_snapshot_copies(self.snapshot_id, [self.zone.id, self.additional_zone.id]) + self.delete_snapshot(self.userapiclient, self.snapshot_id, self.zone.id) + self.verify_snapshot_copies(self.snapshot_id, [self.additional_zone.id]) + return + + @skipTestIf("testsNotSupported") + @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") + def test_04_copy_snapshot_multi_zone_delete_all(self): + """Test to take volume snapshot in a zone, copy in another zone and delete for all + """ + # Validate the following: + # 1. Take snapshot in the native zone + # 2. Copy snapshot in the additional zone + # 3. Verify + # 4. Delete for all zones + # 5. Verify + + snapshot = self.create_snapshot(self.userapiclient, None) + self.snapshot_id = snapshot.id + self.copy_snapshot(self.userapiclient, self.snapshot_id, [str(self.additional_zone.id)], self.zone.id) + self.verify_snapshot_copies(self.snapshot_id, [self.zone.id, self.additional_zone.id]) + self.delete_snapshot(self.userapiclient, self.snapshot_id) + snapshot_entries = Snapshot.list(self.userapiclient, id=snapshot.id) + if snapshot_entries and isinstance(snapshot_entries, list) and len(snapshot_entries) > 0: + self.fail("Snapshot delete for all zones failed") + self.cleanup.remove(snapshot) + return + + @skipTestIf("testsNotSupported") + @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") + def test_05_take_snapshot_multi_zone_create_volume_additional_zone(self): + """Test to take volume snapshot in multiple zones and create a volume in one of the additional zones + """ + # Validate the following: + # 1. Take snapshot in multiple zone + # 2. Verify + # 3. Create volume in the additional zone + # 4. Verify volume zone + + snapshot = self.create_snapshot(self.userapiclient, [str(self.additional_zone.id)]) + self.snapshot_id = snapshot.id + self.verify_snapshot_copies(self.snapshot_id, [self.zone.id, self.additional_zone.id]) + disk_offering_id = None + if snapshot.volumetype == 'ROOT': + service = self.services["disk_offering"] + service["disksize"] = math.ceil(snapshot.virtualsize/(1024*1024*1024)) + self.disk_offering = DiskOffering.create( + self.apiclient, + service + ) + self.cleanup.append(self.disk_offering) + disk_offering_id = self.disk_offering.id + self.volume = self.create_snapshot_volume(self.userapiclient, self.snapshot_id, self.additional_zone.id, disk_offering_id) + if self.additional_zone.id != self.volume.zoneid: + self.fail("Volume from snapshot not created in the additional zone") + return + + @skipTestIf("testsNotSupported") + @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") + def test_06_take_snapshot_multi_zone_create_template_additional_zone(self): + """Test to take volume snapshot in multiple zones and create a volume in one of the additional zones + """ + # Validate the following: + # 1. Take snapshot in multiple zone + # 2. Verify + # 3. Create template in the additional zone + # 4. Verify template zone + + snapshot = self.create_snapshot(self.userapiclient, [str(self.additional_zone.id)]) + self.snapshot_id = snapshot.id + self.verify_snapshot_copies(self.snapshot_id, [self.zone.id, self.additional_zone.id]) + self.template = self.create_snapshot_template(self.userapiclient, self.services, self.snapshot_id, self.additional_zone.id) + if self.additional_zone.id != self.template.zoneid: + self.fail("Template from snapshot not created in the additional zone") + return diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index 04d107e8b869..85f6627a103f 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -51,12 +51,14 @@ "label.action.bulk.delete.isos": "Bulk delete ISOs", "label.action.bulk.delete.load.balancer.rules": "Bulk delete load balancer rules", "label.action.bulk.delete.portforward.rules": "Bulk delete port forward rules", +"label.action.bulk.delete.snapshots": "Bulk delete snapshots", "label.action.bulk.delete.templates": "Bulk delete templates", "label.action.bulk.release.public.ip.address": "Bulk release public IP addresses", "label.action.cancel.maintenance.mode": "Cancel maintenance mode", "label.action.change.password": "Change password", "label.action.configure.stickiness": "Stickiness", "label.action.copy.iso": "Copy ISO", +"label.action.copy.snapshot": "Copy snapshot", "label.action.copy.template": "Copy template", "label.action.create.snapshot.from.vmsnapshot": "Create snapshot from VM snapshot", "label.action.create.template.from.volume": "Create template from volume", @@ -486,6 +488,7 @@ "label.confirm.delete.isos": "Please confirm you wish to delete the selected ISOs.", "label.confirm.delete.loadbalancer.rules": "Please confirm you wish to delete the selected load balancing rules.", "label.confirm.delete.portforward.rules": "Please confirm you wish to delete the selected port-forward rules.", +"label.confirm.delete.snapshot.zones": "Please confirm you wish to delete the snapshot in the selected zones.", "label.confirm.delete.templates": "Please confirm you wish to delete the selected templates.", "label.confirm.delete.tungsten.address.group": "Please confirm that you would like to delete this Address Group", "label.confirm.delete.tungsten.firewall.policy": "Please confirm that you would like to delete this Firewall Policy", @@ -651,6 +654,7 @@ "label.deleting": "Deleting", "label.deleting.failed": "Deleting failed", "label.deleting.iso": "Deleting ISO", +"label.deleting.snapshot": "Deleting snapshot", "label.deleting.template": "Deleting template", "label.demote.project.owner": "Demote account to regular role", "label.demote.project.owner.user": "Demote user to regular role", @@ -2494,6 +2498,8 @@ "message.create.service.offering": "Service offering created.", "message.create.snapshot.from.vmsnapshot.failed": "Failed to create Snapshot from VM Snapshot.", "message.create.snapshot.from.vmsnapshot.progress": "Snapshot creation in progress", +"message.create.template.failed": "Failed to create template.", +"message.create.template.processing": "Template creation in progress", "message.create.volume.failed": "Failed to create volume.", "message.create.volume.processing": "Volume creation in progress", "message.create.vpc.offering": "VPC offering created.", @@ -2942,6 +2948,7 @@ "message.setup.physical.network.during.zone.creation.basic": "When adding a basic zone, you can set up one physical network, which corresponds to a NIC on the hypervisor. The network carries several types of traffic.

You may also add other traffic types onto the physical network.", "message.shared.network.offering.warning": "Domain admins and regular users can only create shared networks from network offering with the setting specifyvlan=false. Please contact an administrator to create a network offering if this list is empty.", "message.shutdown.triggered": "A shutdown has been triggered. CloudStack will not accept new jobs", +"message.snapshot.additional.zones": "Snapshots will always be created in its native zone - %x, here you can select additional zone(s) where it will be copied to at creation time", "message.sourcenatip.change.warning": "WARNING: Changing the sourcenat IP address of the network will cause connectivity downtime for the VMs with NICs in the network.", "message.sourcenatip.change.inhibited": "Changing the sourcenat to this IP of the network to this address is inhibited as firewall rules are defined for it. This can include port forwarding or load balancing rules.\n - If this is an isolated network, please use updateNetwork/click the edit button.\n - If this is a VPC, first clear all other rules for this address.", "message.specify.tag.key": "Please specify a tag key.", @@ -2998,6 +3005,7 @@ "message.success.create.kubernetes.cluter": "Successfully created Kubernetes cluster", "message.success.create.l2.network": "Successfully created L2 network", "message.success.create.snapshot.from.vmsnapshot": "Successfully created snapshot from VM snapshot", +"message.success.create.template": "Successfully created template", "message.success.create.user": "Successfully created user", "message.success.create.volume": "Successfully created volume", "message.success.delete": "Successfully deleted", diff --git a/ui/src/config/section/storage.js b/ui/src/config/section/storage.js index 5a4a88de0d4d..8770f8edc738 100644 --- a/ui/src/config/section/storage.js +++ b/ui/src/config/section/storage.js @@ -318,6 +318,16 @@ export default { name: 'details', component: shallowRef(defineAsyncComponent(() => import('@/components/view/DetailsTab.vue'))) }, + { + name: 'zones', + component: shallowRef(defineAsyncComponent(() => import('@/views/storage/SnapshotZones.vue'))) + }, + { + name: 'events', + resourceType: 'Snapshot', + component: shallowRef(defineAsyncComponent(() => import('@/components/view/EventsTab.vue'))), + show: () => { return 'listEvents' in store.getters.apis } + }, { name: 'comments', component: shallowRef(defineAsyncComponent(() => import('@/components/view/AnnotationsTab.vue'))) @@ -331,12 +341,8 @@ export default { label: 'label.create.template', dataView: true, show: (record) => { return record.state === 'BackedUp' }, - args: ['snapshotid', 'name', 'displaytext', 'ostypeid', 'ispublic', 'isfeatured', 'isdynamicallyscalable', 'requireshvm', 'passwordenabled'], - mapping: { - snapshotid: { - value: (record) => { return record.id } - } - } + popup: true, + component: shallowRef(defineAsyncComponent(() => import('@/views/storage/CreateTemplate.vue'))) }, { api: 'createVolume', diff --git a/ui/src/views/storage/CreateTemplate.vue b/ui/src/views/storage/CreateTemplate.vue new file mode 100644 index 000000000000..294abe330cb6 --- /dev/null +++ b/ui/src/views/storage/CreateTemplate.vue @@ -0,0 +1,294 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + + + + + diff --git a/ui/src/views/storage/CreateVolume.vue b/ui/src/views/storage/CreateVolume.vue index 86d3fcf954a2..3efe31a34e2a 100644 --- a/ui/src/views/storage/CreateVolume.vue +++ b/ui/src/views/storage/CreateVolume.vue @@ -35,7 +35,7 @@ v-model:value="form.name" :placeholder="apiParams.name.description" /> - + @@ -143,6 +143,7 @@ export default { }, data () { return { + snapshotZoneIds: [], zones: [], offerings: [], customDiskOffering: false, @@ -195,10 +196,23 @@ export default { } }, fetchData () { + if (this.createVolumeFromSnapshot) { + this.fetchSnapshotZones() + return + } + let zoneId = null + if (this.createVolumeFromVM) { + zoneId = this.resource.zoneid + } + this.fetchZones(zoneId) + }, + fetchZones (id) { this.loading = true const params = { showicon: true } - if (this.createVolumeFromVM) { - params.id = this.resource.zoneid + if (Array.isArray(id)) { + params.ids = id.join() + } else if (id !== null) { + params.id = id } api('listZones', params).then(json => { this.zones = json.listzonesresponse.zone || [] @@ -208,6 +222,26 @@ export default { this.loading = false }) }, + fetchSnapshotZones () { + this.loading = true + this.snapshotZoneIds = [] + const params = { + showunique: false, + id: this.resource.id + } + api('listSnapshots', params).then(json => { + const snapshots = json.listsnapshotsresponse.snapshot || [] + for (const snapshot of snapshots) { + if (!this.snapshotZoneIds.includes(snapshot.zoneid)) { + this.snapshotZoneIds.push(snapshot.zoneid) + } + } + }).finally(() => { + if (this.snapshotZoneIds && this.snapshotZoneIds.length > 0) { + this.fetchZones(this.snapshotZoneIds) + } + }) + }, fetchDiskOfferings (zoneId) { this.loading = true api('listDiskOfferings', { diff --git a/ui/src/views/storage/FormSchedule.vue b/ui/src/views/storage/FormSchedule.vue index c689de2849d4..ffd89088c4b0 100644 --- a/ui/src/views/storage/FormSchedule.vue +++ b/ui/src/views/storage/FormSchedule.vue @@ -138,6 +138,37 @@ + + + + + + + + + + + + {{ opt.name || opt.description }} + + + + +
{{ $t('label.tags') }}
@@ -194,6 +225,7 @@ import { ref, reactive, toRaw } from 'vue' import { api } from '@/api' import TooltipButton from '@/components/widgets/TooltipButton' +import TooltipLabel from '@/components/widgets/TooltipLabel' import { timeZone } from '@/utils/timezone' import { mixinForm } from '@/utils/mixin' import debounce from 'lodash/debounce' @@ -202,7 +234,8 @@ export default { name: 'FormSchedule', mixins: [mixinForm], components: { - TooltipButton + TooltipButton, + TooltipLabel }, props: { loading: { @@ -216,6 +249,10 @@ export default { resource: { type: Object, required: true + }, + resourceType: { + type: String, + default: null } }, data () { @@ -234,7 +271,8 @@ export default { dayOfMonth: [], timeZoneMap: [], fetching: false, - listDayOfWeek: ['sunday', 'monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday'] + listDayOfWeek: ['sunday', 'monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday'], + zones: [] } }, created () { @@ -242,6 +280,11 @@ export default { this.volumeId = this.resource.id this.fetchTimeZone() }, + computed: { + formattedAdditionalZoneMessage () { + return `${this.$t('message.snapshot.additional.zones').replace('%x', this.resource.zonename)}` + } + }, methods: { initForm () { this.formRef = ref() @@ -262,6 +305,23 @@ export default { maxsnaps: [{ required: true, message: this.$t('message.error.required.input') }], timezone: [{ required: true, message: `${this.$t('message.error.select')}` }] }) + if (this.resourceType === 'Volume') { + this.fetchZoneData() + } + }, + fetchZoneData () { + const params = {} + params.showicon = true + this.zoneLoading = true + api('listZones', params).then(json => { + const listZones = json.listzonesresponse.zone + if (listZones) { + this.zones = listZones + this.zones = this.zones.filter(zone => zone.type !== 'Edge' && zone.id !== this.resource.zoneid) + } + }).finally(() => { + this.zoneLoading = false + }) }, fetchTimeZone (value) { this.timeZoneMap = [] @@ -359,6 +419,9 @@ export default { params.intervaltype = values.intervaltype params.timezone = values.timezone params.maxsnaps = values.maxsnaps + if (values.zoneids && values.zoneids.length > 0) { + params.zoneids = values.zoneids.join() + } switch (values.intervaltype) { case 'hourly': params.schedule = values.time diff --git a/ui/src/views/storage/RecurringSnapshotVolume.vue b/ui/src/views/storage/RecurringSnapshotVolume.vue index 7f4bd5de0293..ce929848de43 100644 --- a/ui/src/views/storage/RecurringSnapshotVolume.vue +++ b/ui/src/views/storage/RecurringSnapshotVolume.vue @@ -23,6 +23,7 @@ :loading="loading" :resource="resource" :dataSource="dataSource" + :resourceType="'Volume'" @close-action="closeAction" @refresh="handleRefresh"/> diff --git a/ui/src/views/storage/ScheduledSnapshots.vue b/ui/src/views/storage/ScheduledSnapshots.vue index 8c95d0b036cc..6792dc310c36 100644 --- a/ui/src/views/storage/ScheduledSnapshots.vue +++ b/ui/src/views/storage/ScheduledSnapshots.vue @@ -61,6 +61,11 @@ + + + + + diff --git a/ui/src/views/storage/TakeSnapshot.vue b/ui/src/views/storage/TakeSnapshot.vue index 7e450b52e164..ee0eafbb56a2 100644 --- a/ui/src/views/storage/TakeSnapshot.vue +++ b/ui/src/views/storage/TakeSnapshot.vue @@ -31,26 +31,47 @@ layout="vertical" @finish="handleSubmit" > - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + {{ opt.name || opt.description }} + + + + + + + + + +
{{ $t('label.tags') }}
@@ -106,12 +127,16 @@ import { ref, reactive, toRaw } from 'vue' import { api } from '@/api' import { mixinForm } from '@/utils/mixin' import TooltipButton from '@/components/widgets/TooltipButton' +import TooltipLabel from '@/components/widgets/TooltipLabel' +import ResourceIcon from '@/components/view/ResourceIcon' export default { name: 'TakeSnapshot', mixins: [mixinForm], components: { - TooltipButton + TooltipButton, + TooltipLabel, + ResourceIcon }, props: { loading: { @@ -131,6 +156,8 @@ export default { inputValue: '', inputKey: '', inputVisible: '', + zones: [], + zoneLoading: false, tags: [], dataSource: [] } @@ -142,6 +169,12 @@ export default { this.initForm() this.quiescevm = this.resource.quiescevm this.supportsStorageSnapshot = this.resource.supportsstoragesnapshot + this.fetchZoneData() + }, + computed: { + formattedAdditionalZoneMessage () { + return `${this.$t('message.snapshot.additional.zones').replace('%x', this.resource.zonename)}` + } }, methods: { initForm () { @@ -153,6 +186,20 @@ export default { }) this.rules = reactive({}) }, + fetchZoneData () { + const params = {} + params.showicon = true + this.zoneLoading = true + api('listZones', params).then(json => { + const listZones = json.listzonesresponse.zone + if (listZones) { + this.zones = listZones + this.zones = this.zones.filter(zone => zone.type !== 'Edge' && zone.id !== this.resource.zoneid) + } + }).finally(() => { + this.zoneLoading = false + }) + }, handleSubmit (e) { e.preventDefault() if (this.actionLoading) return @@ -173,6 +220,9 @@ export default { if (values.quiescevm) { params.quiescevm = values.quiescevm } + if (values.zoneids && values.zoneids.length > 0) { + params.zoneids = values.zoneids.join() + } for (let i = 0; i < this.tags.length; i++) { const formattedTagData = {} const tag = this.tags[i]