Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: implement move resource operation #273

Merged
merged 6 commits into from
Mar 13, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion src/main/java/com/epam/aidial/core/AiDial.java
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
import com.epam.aidial.core.service.InvitationService;
import com.epam.aidial.core.service.LockService;
import com.epam.aidial.core.service.PublicationService;
import com.epam.aidial.core.service.ResourceOperationService;
import com.epam.aidial.core.service.ResourceService;
import com.epam.aidial.core.service.ShareService;
import com.epam.aidial.core.storage.BlobStorage;
Expand Down Expand Up @@ -108,14 +109,15 @@ void start() throws Exception {
InvitationService invitationService = new InvitationService(resourceService, encryptionService, settings("invitations"));
ShareService shareService = new ShareService(resourceService, invitationService, encryptionService);
PublicationService publicationService = new PublicationService(encryptionService, resourceService, storage, generator, clock);
ResourceOperationService resourceOperationService = new ResourceOperationService(resourceService, storage, invitationService, shareService);

AccessService accessService = new AccessService(encryptionService, shareService, publicationService);
RateLimiter rateLimiter = new RateLimiter(vertx, resourceService);

proxy = new Proxy(vertx, client, configStore, logStore,
rateLimiter, upstreamBalancer, accessTokenValidator,
storage, encryptionService, apiKeyStore, tokenStatsTracker, resourceService, invitationService,
shareService, publicationService, accessService, lockService);
shareService, publicationService, accessService, lockService, resourceOperationService);

server = vertx.createHttpServer(new HttpServerOptions(settings("server"))).requestHandler(proxy);
open(server, HttpServer::listen);
Expand Down
2 changes: 2 additions & 0 deletions src/main/java/com/epam/aidial/core/Proxy.java
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import com.epam.aidial.core.service.InvitationService;
import com.epam.aidial.core.service.LockService;
import com.epam.aidial.core.service.PublicationService;
import com.epam.aidial.core.service.ResourceOperationService;
import com.epam.aidial.core.service.ResourceService;
import com.epam.aidial.core.service.ShareService;
import com.epam.aidial.core.storage.BlobStorage;
Expand Down Expand Up @@ -80,6 +81,7 @@ public class Proxy implements Handler<HttpServerRequest> {
private final PublicationService publicationService;
private final AccessService accessService;
private final LockService lockService;
private final ResourceOperationService resourceOperationService;

@Override
public void handle(HttpServerRequest request) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,8 @@ public class ControllerSelector {
private static final Pattern INVITATION = Pattern.compile("^/v1/invitations/([a-zA-Z0-9]+)$");
private static final Pattern PUBLICATIONS = Pattern.compile("^/v1/ops/publications/(list|get|create|delete)$");

private static final Pattern RESOURCE_OPERATIONS = Pattern.compile("^/v1/ops/resources/(move)$");

private static final Pattern DEPLOYMENT_LIMITS = Pattern.compile("^/v1/deployments/([^/]+)/limits$");

public Controller select(Proxy proxy, ProxyContext context) {
Expand Down Expand Up @@ -272,6 +274,12 @@ private static Controller selectPost(Proxy proxy, ProxyContext context, String p
};
}

match = match(RESOURCE_OPERATIONS, path);
if (match != null) {
ResourceOperationController controller = new ResourceOperationController(proxy, context);
return controller::move;
}

return null;
}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
package com.epam.aidial.core.controller;

import com.epam.aidial.core.Proxy;
import com.epam.aidial.core.ProxyContext;
import com.epam.aidial.core.data.MoveResourcesRequest;
import com.epam.aidial.core.security.EncryptionService;
import com.epam.aidial.core.service.LockService;
import com.epam.aidial.core.service.ResourceOperationService;
import com.epam.aidial.core.storage.BlobStorageUtil;
import com.epam.aidial.core.storage.ResourceDescription;
import com.epam.aidial.core.util.HttpException;
import com.epam.aidial.core.util.HttpStatus;
import com.epam.aidial.core.util.ProxyUtil;
import io.vertx.core.Future;
import io.vertx.core.Vertx;
import lombok.extern.slf4j.Slf4j;

@Slf4j
public class ResourceOperationController {

private final ProxyContext context;
private final Proxy proxy;
private final Vertx vertx;
private final EncryptionService encryptionService;
private final ResourceOperationService resourceOperationService;
private final LockService lockService;

public ResourceOperationController(Proxy proxy, ProxyContext context) {
this.context = context;
this.proxy = proxy;
this.vertx = proxy.getVertx();
this.encryptionService = proxy.getEncryptionService();
this.resourceOperationService = proxy.getResourceOperationService();
this.lockService = proxy.getLockService();
}

public Future<?> move() {
context.getRequest()
.body()
.compose(buffer -> {
MoveResourcesRequest request;
try {
request = ProxyUtil.convertToObject(buffer, MoveResourcesRequest.class);
} catch (Exception e) {
log.error("Invalid request body provided", e);
throw new IllegalArgumentException("Can't initiate move resource request. Incorrect body provided");
}

String sourceUrl = request.getSourceUrl();
if (sourceUrl == null) {
throw new IllegalArgumentException("sourceUrl must be provided");
}

String destinationUrl = request.getDestinationUrl();
if (destinationUrl == null) {
throw new IllegalArgumentException("destinationUrl must be provided");
}

String bucketLocation = BlobStorageUtil.buildInitiatorBucket(context);
String bucket = encryptionService.encrypt(bucketLocation);

ResourceDescription sourceResource = ResourceDescription.fromLink(sourceUrl, encryptionService);
if (!sourceResource.getBucketName().equals(bucket)) {
throw new IllegalArgumentException("sourceUrl do not belong to the user");
}

ResourceDescription destinationResource = ResourceDescription.fromLink(destinationUrl, encryptionService);
if (!destinationResource.getBucketName().equals(bucket)) {
throw new IllegalArgumentException("destinationUrl do not belong to the user");
}

if (!sourceResource.getType().equals(destinationResource.getType())) {
throw new IllegalArgumentException("source and destination resources must be the same type");
}


return vertx.executeBlocking(() -> lockService.underBucketLock(proxy, bucketLocation, () -> {
resourceOperationService.moveResource(bucket, bucketLocation, sourceResource, destinationResource, request.isOverwrite());
return null;
}));
})
.onSuccess(ignore -> context.respond(HttpStatus.OK))
.onFailure(this::handleServiceError);

return Future.succeededFuture();
}

private void handleServiceError(Throwable error) {
if (error instanceof IllegalArgumentException) {
context.respond(HttpStatus.BAD_REQUEST, error.getMessage());
} else if (error instanceof HttpException httpException) {
context.respond(httpException.getStatus(), httpException.getMessage());
} else {
context.respond(HttpStatus.INTERNAL_SERVER_ERROR, error.getMessage());
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
package com.epam.aidial.core.data;

import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;

@Data
@AllArgsConstructor
@NoArgsConstructor
public class MoveResourcesRequest {
String sourceUrl;
String destinationUrl;
boolean overwrite;
}
5 changes: 5 additions & 0 deletions src/main/java/com/epam/aidial/core/data/SharedByMeDto.java
Original file line number Diff line number Diff line change
Expand Up @@ -18,4 +18,9 @@ public void addUserToResource(String url, String userLocation) {
Set<String> users = resourceToUsers.computeIfAbsent(url, k -> new HashSet<>());
users.add(userLocation);
}

public void addUsersToResource(String url, Set<String> userLocations) {
Set<String> users = resourceToUsers.computeIfAbsent(url, k -> new HashSet<>());
users.addAll(userLocations);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,27 @@ public void cleanUpResourceLinks(String bucket, String location, Set<ResourceLin
});
}

public void moveResource(String bucket, String location, ResourceDescription source, ResourceDescription destination) {
ResourceDescription resource = ResourceDescription.fromDecoded(ResourceType.INVITATION, bucket, location, INVITATION_RESOURCE_FILENAME);
ResourceLink sourceLink = new ResourceLink(source.getUrl());
ResourceLink destinationLink = new ResourceLink(destination.getUrl());
resourceService.computeResource(resource, state -> {
InvitationsMap invitations = ProxyUtil.convertToObject(state, InvitationsMap.class);
if (invitations == null) {
return null;
}
Map<String, Invitation> invitationMap = invitations.getInvitations();
for (Invitation invitation : invitationMap.values()) {
Set<ResourceLink> invitationResourceLinks = invitation.getResources();
if (invitationResourceLinks.remove(sourceLink)) {
invitationResourceLinks.add(destinationLink);
}
}

return ProxyUtil.convertToString(invitations);
});
}

private void cleanUpExpiredInvitations(ResourceDescription resource, Collection<String> idsToEvict) {
resourceService.computeResource(resource, state -> {
InvitationsMap invitations = ProxyUtil.convertToObject(state, InvitationsMap.class);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
package com.epam.aidial.core.service;

import com.epam.aidial.core.data.ResourceType;
import com.epam.aidial.core.storage.BlobStorage;
import com.epam.aidial.core.storage.ResourceDescription;
import lombok.AllArgsConstructor;

@AllArgsConstructor
public class ResourceOperationService {

private final ResourceService resourceService;
private final BlobStorage storage;
private final InvitationService invitationService;
private final ShareService shareService;

public void moveResource(String bucket, String location, ResourceDescription source, ResourceDescription destination, boolean overwriteIfExists) {
if (source.isFolder() || destination.isFolder()) {
throw new IllegalArgumentException("Moving folders is not supported");
}

String sourceResourcePath = source.getAbsoluteFilePath();
String sourceResourceUrl = source.getUrl();
String destinationResourcePath = destination.getAbsoluteFilePath();
String destinationResourceUrl = destination.getUrl();

if (!hasResource(source)) {
throw new IllegalArgumentException("Source resource %s do not exists".formatted(sourceResourceUrl));
}

ResourceType resourceType = source.getType();
switch (resourceType) {
case FILE -> {
if (!overwriteIfExists && storage.exists(destinationResourcePath)) {
throw new IllegalArgumentException("Can't move resource %s to %s, because destination resource already exists"
.formatted(sourceResourceUrl, destinationResourceUrl));
}
storage.copy(sourceResourcePath, destinationResourcePath);
storage.delete(sourceResourcePath);
}
case CONVERSATION, PROMPT -> {
boolean copied = resourceService.copyResource(source, destination, overwriteIfExists);
if (!copied) {
throw new IllegalArgumentException("Can't move resource %s to %s, because destination resource already exists"
.formatted(sourceResourceUrl, destinationResourceUrl));
}
resourceService.deleteResource(source);
}
default -> throw new IllegalArgumentException("Unsupported resource type " + resourceType);
}
// move source links to destination if any
invitationService.moveResource(bucket, location, source, destination);
// move shared access if any
shareService.moveSharedAccess(bucket, location, source, destination);
}

private boolean hasResource(ResourceDescription resource) {
return switch (resource.getType()) {
case FILE -> storage.exists(resource.getAbsoluteFilePath());
case CONVERSATION, PROMPT -> resourceService.hasResource(resource);
default -> throw new IllegalArgumentException("Unsupported resource type " + resource.getType());
};
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -278,14 +278,18 @@ public boolean deleteResource(ResourceDescription descriptor) {
}

public boolean copyResource(ResourceDescription from, ResourceDescription to) {
return copyResource(from, to, true);
}

public boolean copyResource(ResourceDescription from, ResourceDescription to, boolean overwrite) {
String body = getResource(from);

if (body == null) {
return false;
}

putResource(to, body, true);
return true;
ResourceItemMetadata metadata = putResource(to, body, overwrite);
return metadata != null;
}

private Void sync() {
Expand Down
54 changes: 54 additions & 0 deletions src/main/java/com/epam/aidial/core/service/ShareService.java
Original file line number Diff line number Diff line change
Expand Up @@ -303,6 +303,48 @@ public void discardSharedAccess(String bucket, String location, ResourceLinkColl
}
}

public void copySharedAccess(String bucket, String location, ResourceDescription source, ResourceDescription destination) {
ResourceType sourceResourceType = source.getType();
ResourceDescription sharedByMeResource = getShareResource(ResourceType.SHARED_BY_ME, sourceResourceType, bucket, location);
SharedByMeDto sharedByMeDto = ProxyUtil.convertToObject(resourceService.getResource(sharedByMeResource), SharedByMeDto.class);
if (sharedByMeDto == null) {
return;
}

Set<String> userLocations = sharedByMeDto.getResourceToUsers().get(source.getUrl());

ResourceType destinationResourceType = destination.getType();
String destinationResourceLink = destination.getUrl();
// source and destination resource type might be different
astsiapanay marked this conversation as resolved.
Show resolved Hide resolved
sharedByMeResource = getShareResource(ResourceType.SHARED_BY_ME, destinationResourceType, bucket, location);

// copy user locations from source to destination
resourceService.computeResource(sharedByMeResource, state -> {
SharedByMeDto dto = ProxyUtil.convertToObject(state, SharedByMeDto.class);
if (dto == null) {
dto = new SharedByMeDto(new HashMap<>());
}

// add shared access to the destination resource
dto.addUsersToResource(destinationResourceLink, userLocations);

return ProxyUtil.convertToString(dto);
});

// add each user shared access to the destination resource
for (String userLocation : userLocations) {
String userBucket = encryptionService.encrypt(userLocation);
addSharedResource(userBucket, userLocation, destinationResourceLink, destinationResourceType);
}
}

public void moveSharedAccess(String bucket, String location, ResourceDescription source, ResourceDescription destination) {
// copy shared access from source to destination
copySharedAccess(bucket, location, source, destination);
// revoke shared access from source
revokeSharedAccess(bucket, location, new ResourceLinkCollection(Set.of(new ResourceLink(source.getUrl()))));
}

private void removeSharedResource(String bucket, String location, String link, ResourceType resourceType) {
ResourceDescription sharedByMeResource = getShareResource(ResourceType.SHARED_WITH_ME, resourceType, bucket, location);
resourceService.computeResource(sharedByMeResource, state -> {
Expand All @@ -315,6 +357,18 @@ private void removeSharedResource(String bucket, String location, String link, R
});
}

private void addSharedResource(String bucket, String location, String link, ResourceType resourceType) {
ResourceDescription sharedByMeResource = getShareResource(ResourceType.SHARED_WITH_ME, resourceType, bucket, location);
resourceService.computeResource(sharedByMeResource, state -> {
ResourceLinkCollection sharedWithMe = ProxyUtil.convertToObject(state, ResourceLinkCollection.class);
if (sharedWithMe != null) {
sharedWithMe.getResources().add(new ResourceLink(link));
}

return ProxyUtil.convertToString(sharedWithMe);
});
}

private List<MetadataBase> linksToMetadata(Stream<String> links) {
return links
.map(link -> ResourceDescription.fromLink(link, encryptionService))
Expand Down
Loading
Loading