Skip to content

Commit

Permalink
api,server,ui: snapshot copy, multi-zone replica (apache#7873)
Browse files Browse the repository at this point in the history
This PR adds new functionality to copy snapshots across zones and take snapshots for multiple zones.

Copy functionality is similar to template copy. The source zone acts as the web server from where the destination zone(s) can download the snapshot files. For this purpose, a new API - `copySnapshot` has been added. The response for copySnapshot will be returning zone and download details from the first destination zone of the request. This behaviour is similar to the `copyTemplate` API.

In a similar manner, multiple zones can be selected while taking the snapshots or creating snapshot policies. For this snapshot will be taken in the base zone(in which volume is present) and then copied to the additional zones. A new parameter - `zoneids` has been added to `createSnapshot` and `createSnapshotPolicy` APIs.

As snapshots can be present on multiple zones (secondary stores), a new parameter `zoneid` has been added to delete the snapshot copy on a specific zone.

`listSnapshots` API has been updated to allow listing snapshot entries for different zones/datastores. New parameters - `showUnique`, `locationType` have been added.

Events generated during snapshot operations will now be linked to the snapshot itself rather than the volume of the snapshot.

`listSnapshotPolicies` and `createSnapshotPolicy` APIs will return zone details of the zones in which backup will be scheduled for the policy.

----
New API added
`copySnapshot`

Request and response params updated for APIs
```
- listSnapshots
- deleteSnapshot
- createTemplate
- listZones
- listSnapshotPolicies
- createSnapshotPolicy
```
UI updated for
- Snapshot detail view
- Create snapshot form
- Create snapshot policy form
- Create volume (from snapshot) form
- Create template (from snapshot) form

Doc PR: apache/cloudstack-documentation#344
PR: apache#7873
  • Loading branch information
shwstppr authored Oct 23, 2023
1 parent 99ded81 commit 543c54c
Show file tree
Hide file tree
Showing 132 changed files with 7,231 additions and 1,105 deletions.
3 changes: 2 additions & 1 deletion api/src/main/java/com/cloud/event/EventTypes.java
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
import org.apache.cloudstack.config.Configuration;
import org.apache.cloudstack.ha.HAConfig;
import org.apache.cloudstack.usage.Usage;
import org.apache.cloudstack.vm.schedule.VMSchedule;

import com.cloud.dc.DataCenter;
import com.cloud.dc.DataCenterGuestIpv6Prefix;
Expand Down Expand Up @@ -84,7 +85,6 @@
import com.cloud.vm.Nic;
import com.cloud.vm.NicSecondaryIp;
import com.cloud.vm.VirtualMachine;
import org.apache.cloudstack.vm.schedule.VMSchedule;

public class EventTypes {

Expand Down Expand Up @@ -320,6 +320,7 @@ public class EventTypes {
public static final String EVENT_DOMAIN_UPDATE = "DOMAIN.UPDATE";

// Snapshots
public static final String EVENT_SNAPSHOT_COPY = "SNAPSHOT.COPY";
public static final String EVENT_SNAPSHOT_CREATE = "SNAPSHOT.CREATE";
public static final String EVENT_SNAPSHOT_ON_PRIMARY = "SNAPSHOT.ON_PRIMARY";
public static final String EVENT_SNAPSHOT_OFF_PRIMARY = "SNAPSHOT.OFF_PRIMARY";
Expand Down
7 changes: 4 additions & 3 deletions api/src/main/java/com/cloud/storage/VolumeApiService.java
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,9 @@
package com.cloud.storage;

import java.net.MalformedURLException;
import java.util.List;
import java.util.Map;

import com.cloud.utils.fsm.NoTransitionException;
import org.apache.cloudstack.api.command.user.volume.AssignVolumeCmd;
import org.apache.cloudstack.api.command.user.volume.AttachVolumeCmd;
import org.apache.cloudstack.api.command.user.volume.ChangeOfferingForVolumeCmd;
Expand All @@ -37,6 +37,7 @@

import com.cloud.exception.ResourceAllocationException;
import com.cloud.user.Account;
import com.cloud.utils.fsm.NoTransitionException;

public interface VolumeApiService {

Expand Down Expand Up @@ -105,10 +106,10 @@ public interface VolumeApiService {

Volume detachVolumeFromVM(DetachVolumeCmd cmd);

Snapshot takeSnapshot(Long volumeId, Long policyId, Long snapshotId, Account account, boolean quiescevm, Snapshot.LocationType locationType, boolean asyncBackup, Map<String, String> tags)
Snapshot takeSnapshot(Long volumeId, Long policyId, Long snapshotId, Account account, boolean quiescevm, Snapshot.LocationType locationType, boolean asyncBackup, Map<String, String> tags, List<Long> zoneIds)
throws ResourceAllocationException;

Snapshot allocSnapshot(Long volumeId, Long policyId, String snapshotName, Snapshot.LocationType locationType) throws ResourceAllocationException;
Snapshot allocSnapshot(Long volumeId, Long policyId, String snapshotName, Snapshot.LocationType locationType, List<Long> zoneIds) throws ResourceAllocationException;

Volume updateVolume(long volumeId, String path, String state, Long storageId, Boolean displayVolume, String customId, long owner, String chainInfo, String name);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,18 +18,20 @@

import java.util.List;

import org.apache.cloudstack.api.command.user.snapshot.CopySnapshotCmd;
import org.apache.cloudstack.api.command.user.snapshot.CreateSnapshotPolicyCmd;
import org.apache.cloudstack.api.command.user.snapshot.DeleteSnapshotPoliciesCmd;
import org.apache.cloudstack.api.command.user.snapshot.ListSnapshotPoliciesCmd;
import org.apache.cloudstack.api.command.user.snapshot.ListSnapshotsCmd;
import org.apache.cloudstack.api.command.user.snapshot.UpdateSnapshotPolicyCmd;

import com.cloud.api.commands.ListRecurringSnapshotScheduleCmd;
import com.cloud.exception.ResourceAllocationException;
import com.cloud.exception.StorageUnavailableException;
import com.cloud.storage.Snapshot;
import com.cloud.storage.Volume;
import com.cloud.user.Account;
import com.cloud.utils.Pair;
import org.apache.cloudstack.api.command.user.snapshot.UpdateSnapshotPolicyCmd;

public interface SnapshotApiService {

Expand All @@ -50,7 +52,7 @@ public interface SnapshotApiService {
* @param snapshotId
* TODO
*/
boolean deleteSnapshot(long snapshotId);
boolean deleteSnapshot(long snapshotId, Long zoneId);

/**
* Creates a policy with specified schedule. maxSnaps specifies the number of most recent snapshots that are to be
Expand Down Expand Up @@ -88,7 +90,7 @@ public interface SnapshotApiService {

Snapshot allocSnapshot(Long volumeId, Long policyId, String snapshotName, Snapshot.LocationType locationType) throws ResourceAllocationException;

Snapshot allocSnapshot(Long volumeId, Long policyId, String snapshotName, Snapshot.LocationType locationType, Boolean isFromVmSnapshot)
Snapshot allocSnapshot(Long volumeId, Long policyId, String snapshotName, Snapshot.LocationType locationType, Boolean isFromVmSnapshot, List<Long> zoneIds)
throws ResourceAllocationException;


Expand Down Expand Up @@ -124,4 +126,6 @@ Snapshot allocSnapshot(Long volumeId, Long policyId, String snapshotName, Snapsh
SnapshotPolicy updateSnapshotPolicy(UpdateSnapshotPolicyCmd updateSnapshotPolicyCmd);

void markVolumeSnapshotsAsDestroyed(Volume volume);

Snapshot copySnapshot(CopySnapshotCmd cmd) throws StorageUnavailableException, ResourceAllocationException;
}
3 changes: 3 additions & 0 deletions api/src/main/java/org/apache/cloudstack/api/ApiConstants.java
Original file line number Diff line number Diff line change
Expand Up @@ -76,8 +76,10 @@ public class ApiConstants {
public static final String CSR = "csr";
public static final String PRIVATE_KEY = "privatekey";
public static final String DATASTORE_HOST = "datastorehost";
public static final String DATASTORE_ID = "datastoreid";
public static final String DATASTORE_NAME = "datastorename";
public static final String DATASTORE_PATH = "datastorepath";
public static final String DATASTORE_STATE = "datastorestate";
public static final String DATASTORE_TYPE = "datastoretype";
public static final String DOMAIN_SUFFIX = "domainsuffix";
public static final String DNS_SEARCH_ORDER = "dnssearchorder";
Expand Down Expand Up @@ -492,6 +494,7 @@ public class ApiConstants {
public static final String ZONE = "zone";
public static final String ZONE_ID = "zoneid";
public static final String ZONE_NAME = "zonename";
public static final String ZONE_WISE = "zonewise";
public static final String NETWORK_TYPE = "networktype";
public static final String PAGE = "page";
public static final String PAGE_SIZE = "pagesize";
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,181 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

package org.apache.cloudstack.api.command.user.snapshot;

import java.util.ArrayList;
import java.util.List;

import org.apache.cloudstack.acl.RoleType;
import org.apache.cloudstack.api.APICommand;
import org.apache.cloudstack.api.ApiCommandResourceType;
import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.api.ApiErrorCode;
import org.apache.cloudstack.api.BaseAsyncCmd;
import org.apache.cloudstack.api.Parameter;
import org.apache.cloudstack.api.ResponseObject;
import org.apache.cloudstack.api.ServerApiException;
import org.apache.cloudstack.api.command.user.UserCmd;
import org.apache.cloudstack.api.response.SnapshotResponse;
import org.apache.cloudstack.api.response.ZoneResponse;
import org.apache.cloudstack.context.CallContext;
import org.apache.commons.collections.CollectionUtils;
import org.apache.log4j.Logger;

import com.cloud.dc.DataCenter;
import com.cloud.event.EventTypes;
import com.cloud.exception.ResourceAllocationException;
import com.cloud.exception.ResourceUnavailableException;
import com.cloud.exception.StorageUnavailableException;
import com.cloud.storage.Snapshot;
import com.cloud.user.Account;

@APICommand(name = "copySnapshot", description = "Copies a snapshot from one zone to another.",
responseObject = SnapshotResponse.class, responseView = ResponseObject.ResponseView.Restricted,
requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, since = "4.19.0",
authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User})
public class CopySnapshotCmd extends BaseAsyncCmd implements UserCmd {
public static final Logger s_logger = Logger.getLogger(CopySnapshotCmd.class.getName());

/////////////////////////////////////////////////////
//////////////// API parameters /////////////////////
/////////////////////////////////////////////////////

@Parameter(name = ApiConstants.ID, type = CommandType.UUID,
entityType = SnapshotResponse.class, required = true, description = "the ID of the snapshot.")
private Long id;

@Parameter(name = ApiConstants.SOURCE_ZONE_ID,
type = CommandType.UUID,
entityType = ZoneResponse.class,
description = "The ID of the zone in which the snapshot is currently present. " +
"If not specified then the zone of snapshot's volume will be used.")
private Long sourceZoneId;

@Parameter(name = ApiConstants.DESTINATION_ZONE_ID,
type = CommandType.UUID,
entityType = ZoneResponse.class,
required = false,
description = "The ID of the zone the snapshot is being copied to.")
protected Long destZoneId;

@Parameter(name = ApiConstants.DESTINATION_ZONE_ID_LIST,
type=CommandType.LIST,
collectionType = CommandType.UUID,
entityType = ZoneResponse.class,
required = false,
description = "A comma-separated list of IDs of the zones that the snapshot needs to be copied to. " +
"Specify this list if the snapshot needs to copied to multiple zones in one go. " +
"Do not specify destzoneid and destzoneids together, however one of them is required.")
protected List<Long> destZoneIds;

/////////////////////////////////////////////////////
/////////////////// Accessors ///////////////////////
/////////////////////////////////////////////////////


public Long getId() {
return id;
}

public Long getSourceZoneId() {
return sourceZoneId;
}

public List<Long> getDestinationZoneIds() {
if (destZoneIds != null && destZoneIds.size() != 0) {
return destZoneIds;
}
if (destZoneId != null) {
List < Long > destIds = new ArrayList<>();
destIds.add(destZoneId);
return destIds;
}
return null;
}

@Override
public String getEventType() {
return EventTypes.EVENT_SNAPSHOT_COPY;
}

@Override
public String getEventDescription() {
StringBuilder descBuilder = new StringBuilder();
if (getDestinationZoneIds() != null) {
for (Long destId : getDestinationZoneIds()) {
descBuilder.append(", ");
descBuilder.append(_uuidMgr.getUuid(DataCenter.class, destId));
}
if (descBuilder.length() > 0) {
descBuilder.deleteCharAt(0);
}
}

return "copying snapshot: " + _uuidMgr.getUuid(Snapshot.class, getId()) + ((descBuilder.length() > 0) ? " to zones: " + descBuilder.toString() : "");
}

@Override
public ApiCommandResourceType getApiResourceType() {
return ApiCommandResourceType.Snapshot;
}

@Override
public Long getApiResourceId() {
return getId();
}

@Override
public long getEntityOwnerId() {
Snapshot snapshot = _entityMgr.findById(Snapshot.class, getId());
if (snapshot != null) {
return snapshot.getAccountId();
}
return Account.ACCOUNT_ID_SYSTEM;
}

@Override
public void execute() throws ResourceUnavailableException {
try {
if (destZoneId == null && CollectionUtils.isEmpty(destZoneIds))
throw new ServerApiException(ApiErrorCode.PARAM_ERROR,
"Either destzoneid or destzoneids parameters have to be specified.");

if (destZoneId != null && CollectionUtils.isNotEmpty(destZoneIds))
throw new ServerApiException(ApiErrorCode.PARAM_ERROR,
"Both destzoneid and destzoneids cannot be specified at the same time.");

CallContext.current().setEventDetails(getEventDescription());
Snapshot snapshot = _snapshotService.copySnapshot(this);

if (snapshot != null) {
SnapshotResponse response = _queryService.listSnapshot(this);
response.setResponseName(getCommandName());
setResponseObject(response);
} else {
throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to copy snapshot");
}
} catch (StorageUnavailableException ex) {
s_logger.warn("Exception: ", ex);
throw new ServerApiException(ApiErrorCode.RESOURCE_UNAVAILABLE_ERROR, ex.getMessage());
} catch (ResourceAllocationException ex) {
s_logger.warn("Exception: ", ex);
throw new ServerApiException(ApiErrorCode.RESOURCE_ALLOCATION_ERROR, ex.getMessage());
}

}
}
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.cloudstack.api.APICommand;
Expand All @@ -32,6 +33,7 @@
import org.apache.cloudstack.api.response.SnapshotPolicyResponse;
import org.apache.cloudstack.api.response.SnapshotResponse;
import org.apache.cloudstack.api.response.VolumeResponse;
import org.apache.cloudstack.api.response.ZoneResponse;
import org.apache.commons.collections.MapUtils;
import org.apache.log4j.Logger;

Expand Down Expand Up @@ -90,6 +92,15 @@ public class CreateSnapshotCmd extends BaseAsyncCreateCmd {
@Parameter(name = ApiConstants.TAGS, type = CommandType.MAP, description = "Map of tags (key/value pairs)")
private Map tags;

@Parameter(name = ApiConstants.ZONE_ID_LIST,
type=CommandType.LIST,
collectionType = CommandType.UUID,
entityType = ZoneResponse.class,
description = "A comma-separated list of IDs of the zones in which the snapshot will be made available. " +
"The snapshot will always be made available in the zone in which the volume is present.",
since = "4.19.0")
protected List<Long> zoneIds;

private String syncObjectType = BaseAsyncCmd.snapshotHostSyncObject;

// ///////////////////////////////////////////////////
Expand Down Expand Up @@ -148,6 +159,10 @@ private Long getHostId() {
return _snapshotService.getHostIdForSnapshotOperation(volume);
}

public List<Long> getZoneIds() {
return zoneIds;
}

// ///////////////////////////////////////////////////
// ///////////// API Implementation///////////////////
// ///////////////////////////////////////////////////
Expand Down Expand Up @@ -196,7 +211,7 @@ public ApiCommandResourceType getApiResourceType() {

@Override
public void create() throws ResourceAllocationException {
Snapshot snapshot = _volumeService.allocSnapshot(getVolumeId(), getPolicyId(), getSnapshotName(), getLocationType());
Snapshot snapshot = _volumeService.allocSnapshot(getVolumeId(), getPolicyId(), getSnapshotName(), getLocationType(), getZoneIds());
if (snapshot != null) {
setEntityId(snapshot.getId());
setEntityUuid(snapshot.getUuid());
Expand All @@ -210,7 +225,7 @@ public void execute() {
Snapshot snapshot;
try {
snapshot =
_volumeService.takeSnapshot(getVolumeId(), getPolicyId(), getEntityId(), _accountService.getAccount(getEntityOwnerId()), getQuiescevm(), getLocationType(), getAsyncBackup(), getTags());
_volumeService.takeSnapshot(getVolumeId(), getPolicyId(), getEntityId(), _accountService.getAccount(getEntityOwnerId()), getQuiescevm(), getLocationType(), getAsyncBackup(), getTags(), getZoneIds());

if (snapshot != null) {
SnapshotResponse response = _responseGenerator.createSnapshotResponse(snapshot);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -186,7 +186,7 @@ public void execute() {
} finally {
if (snapshot == null) {
try {
_snapshotService.deleteSnapshot(getEntityId());
_snapshotService.deleteSnapshot(getEntityId(), null);
} catch (Exception e) {
s_logger.debug("Failed to clean failed snapshot" + getEntityId());
}
Expand Down
Loading

0 comments on commit 543c54c

Please sign in to comment.