Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[#186091347] Create endpoints to clean up a run #13

Merged
merged 8 commits into from
Oct 27, 2023
Merged

[#186091347] Create endpoints to clean up a run #13

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
a051f8b
Create new endpoint to get run files
Sean-DNAstack Oct 16, 2023
bdb534a
Create new endpoint to delete run files
Sean-DNAstack Oct 18, 2023
18b45a1
Address PR feedback
Sean-DNAstack Oct 19, 2023
dc8436a
Address PR feedback
Sean-DNAstack Oct 20, 2023
6d27bac
Added e2e tests
Sean-DNAstack Oct 20, 2023
f4f0642
Add extra file to wdl
Sean-DNAstack Oct 20, 2023
92869fc
Clean up wes swagger yaml
Sean-DNAstack Oct 23, 2023
15038c1
Optimize & clean up e2e tests
Sean-DNAstack Oct 24, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
152 changes: 150 additions & 2 deletions e2e-tests/src/main/java/com/dnastack/wes/service/WesE2ETest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
import com.google.auth.oauth2.GoogleCredentials;
import io.restassured.builder.MultiPartSpecBuilder;
import io.restassured.http.ContentType;
import io.restassured.http.Header;
import io.restassured.specification.MultiPartSpecification;
import org.awaitility.core.ConditionFactory;
import org.junit.jupiter.api.*;
Expand All @@ -20,7 +19,6 @@
import java.util.*;

import static io.restassured.RestAssured.given;
import static java.lang.String.format;
import static org.awaitility.Awaitility.with;
import static org.hamcrest.Matchers.*;

Expand Down Expand Up @@ -433,6 +431,7 @@ public void uploadWorkflowAttachmentWithRunSubmission() throws Exception {
public class RunMethodsWithExistingJobs {

String workflowJobId;
String workflowJobIdWithAllOutputTypes;

@BeforeAll
public void setup() throws InterruptedException {
Expand All @@ -452,6 +451,22 @@ public void setup() throws InterruptedException {
.multiPart(getJsonMultipart("tags", tags))
.multiPart(getJsonMultipart("workflow_params", inputs))
.post(path)
.then()
.assertThat()
.statusCode(200)
.body("run_id",is(notNullValue()))
.extract()
.jsonPath()
.getString("run_id");

workflowJobIdWithAllOutputTypes = given()
.log().uri()
.log().method()
.header(getHeader(getResource(path)))
.multiPart(getWorkflowUrlMultipart("echo.wdl"))
.multiPart(getMultipartAttachment("echo.wdl",supplier.getFileContent(WdlSupplier.WORKFLOW_WITH_ALL_OUTPUT_TYPES).getBytes()))
.multiPart(getJsonMultipart("workflow_params", inputs))
.post(path)
.then()
.assertThat()
.statusCode(200)
Expand Down Expand Up @@ -569,6 +584,139 @@ public void listRunsReturnsReturnsNonEmptyCollection() {
//@formatter:on
}


@Test
@DisplayName("Get Run Files for existing run returns all files")
public void getRunFilesReturnsNonEmptyCollection() throws Exception {
String path = getRootPath() + "/runs/" + workflowJobIdWithAllOutputTypes + "/files";
pollUntilJobCompletes(workflowJobIdWithAllOutputTypes);
Sean-DNAstack marked this conversation as resolved.
Show resolved Hide resolved

//@formatter:off
given()
.log().uri()
.log().method()
.header(getHeader(getResource(getRootPath() + "/runs/" + workflowJobIdWithAllOutputTypes)))
.accept(ContentType.JSON)
.get(path)
.then()
.assertThat()
.statusCode(200)
.body("runFiles.size()", greaterThan(0))
.body("runFiles.every { it.path != null && it.file_type in ['FINAL', 'SECONDARY', 'LOG'] }", equalTo(true));
//@formatter:on
}


@Test
@DisplayName("Get Run Files for non-existent run fails with status 401 or 404")
public void getRunFilesForNonExistentRunShouldFail() {
String resourcePath = getRootPath() + "/runs/" + UUID.randomUUID();
String path = resourcePath + "/files";

//@formatter:off
given()
.log().uri()
.log().method()
.header(getHeader(getResource(resourcePath)))
.accept(ContentType.JSON)
.get(path)
.then()
.assertThat()
.statusCode(anyOf(equalTo(404), equalTo(401)));
//@formatter:on
}


@Test
@DisplayName("Delete Run Files for existing run returns all deleted files")
public void deleteRunFilesReturnsNonEmptyCollection() throws Exception {
reRunWorkflowWithAllOutputTypes();
String path = getRootPath() + "/runs/" + workflowJobIdWithAllOutputTypes + "/files";
pollUntilJobCompletes(workflowJobIdWithAllOutputTypes);
Sean-DNAstack marked this conversation as resolved.
Show resolved Hide resolved

//@formatter:off
given()
.log().uri()
.log().method()
.header(getHeader(getResource(getRootPath() + "/runs/" + workflowJobIdWithAllOutputTypes)))
.accept(ContentType.JSON)
.delete(path)
.then()
.assertThat()
.statusCode(200)
.body("deletions.size()", greaterThan(0))
.body("deletions.every { it.path != null && it.file_type == 'SECONDARY' && it.state == 'DELETED' }", equalTo(true));
Sean-DNAstack marked this conversation as resolved.
Show resolved Hide resolved
//@formatter:on
}


@Test
@DisplayName("Delete Run Files for existing run asynchronously returns all deleted files")
public void deleteRunFilesAsyncReturnsNonEmptyCollection() throws Exception {
reRunWorkflowWithAllOutputTypes();
String path = getRootPath() + "/runs/" + workflowJobIdWithAllOutputTypes + "/files";
pollUntilJobCompletes(workflowJobIdWithAllOutputTypes);
Sean-DNAstack marked this conversation as resolved.
Show resolved Hide resolved

//@formatter:off
given()
.log().uri()
.log().method()
.header(getHeader(getResource(getRootPath() + "/runs/" + workflowJobIdWithAllOutputTypes)))
.accept(ContentType.JSON)
.queryParam("async", true)
.delete(path)
.then()
.assertThat()
.statusCode(200)
.body("deletions.size()", greaterThan(0))
.body("deletions.every { it.path != null && it.file_type == 'SECONDARY' && it.state == 'ASYNC' }", equalTo(true));
//@formatter:on
Sean-DNAstack marked this conversation as resolved.
Show resolved Hide resolved
}


@Test
@DisplayName("Delete Run Files for non-existent run fails with status 401 or 404")
public void deleteRunFilesForNonExistentRunShouldFail() {
String resourcePath = getRootPath() + "/runs/" + UUID.randomUUID();
String path = resourcePath + "/files";

//@formatter:off
given()
.log().uri()
.log().method()
.header(getHeader(getResource(resourcePath)))
.accept(ContentType.JSON)
.delete(path)
.then()
.assertThat()
.statusCode(anyOf(equalTo(404),equalTo(401)));
//@formatter:on
}


private void reRunWorkflowWithAllOutputTypes() {
Sean-DNAstack marked this conversation as resolved.
Show resolved Hide resolved
String path = getRootPath() + "/runs";
Map<String, String> inputs = Collections.singletonMap("hello_world.name", "Some sort of String");

//@formatter:off
workflowJobIdWithAllOutputTypes = given()
.log().uri()
.log().method()
.header(getHeader(getResource(path)))
.multiPart(getWorkflowUrlMultipart("echo.wdl"))
.multiPart(getMultipartAttachment("echo.wdl",supplier.getFileContent(WdlSupplier.WORKFLOW_WITH_ALL_OUTPUT_TYPES).getBytes()))
.multiPart(getJsonMultipart("workflow_params", inputs))
.post(path)
.then()
.assertThat()
.statusCode(200)
.body("run_id",is(notNullValue()))
.extract()
.jsonPath()
.getString("run_id");
//@formatter:on
}

}

}
Expand Down
1 change: 1 addition & 0 deletions e2e-tests/src/main/java/com/dnastack/wes/service/wdl/WdlSupplier.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
public class WdlSupplier {

public static final String WORKFLOW_WITHOUT_FILE = "workflow_without_file.wdl";
public static final String WORKFLOW_WITH_ALL_OUTPUT_TYPES = "workflow_with_all_output_types.wdl";
public static final String WORKFLOW_WITH_IMPORTS_1 = "workflow_with_imports_1.wdl";
public static final String WORKFLOW_WITH_IMPORTS_2 = "workflow_with_imports_2.wdl";
public static final String WORKFLOW_WITH_IMPORTS_INPUTS = "workflow_with_imports.json";
Expand Down
30 changes: 30 additions & 0 deletions e2e-tests/src/main/resources/com/dnastack/wes/service/wdl/workflow_with_all_output_types.wdl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
task echo {
String name
command {
echo "Hello ${name}"
>&2 echo "Goodbye ${name}"
echo "Bye" > "test.txt"
echo "Bye" > "test2.txt"
}

runtime {
docker: "ubuntu"
}

output {
File out = stdout()
File out2 = "test.txt"
Array[File] arrayOut = [out, out2, "test2.txt"]
}
}

workflow hello_world {
String name
call echo {
input:
name = name
}
output {
File out = echo.out
}
}
3 changes: 2 additions & 1 deletion src/main/java/com/dnastack/wes/config/AsyncConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.task.TaskExecutor;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;

import java.util.concurrent.ThreadPoolExecutor;
Expand All @@ -13,7 +14,7 @@
public class AsyncConfig {

@Bean
public ThreadPoolTaskExecutor defaultAsyncOperationExecutor(
public TaskExecutor defaultAsyncOperationExecutor(
@Value("${app.executors.default.core-pool-size:8}") int corePoolSize,
@Value("${app.executors.default.max-pool-size:16}") int maxPoolSize,
@Value("${app.executors.default.queue-capacity:5000}") int queueCapacity
Expand Down
36 changes: 17 additions & 19 deletions src/main/java/com/dnastack/wes/cromwell/CromwellService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,8 @@
import feign.FeignException;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.core.task.TaskExecutor;
import org.springframework.http.HttpRange;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;

Expand Down Expand Up @@ -55,7 +54,7 @@ public class CromwellService {
private final PathTranslatorFactory pathTranslatorFactory;
private final CromwellWesMapper cromwellWesMapper;
private final CromwellConfig cromwellConfig;
private final ThreadPoolTaskExecutor executor;
private final TaskExecutor defaultAsyncOperationExecutor;

private final AppConfig appConfig;

Expand All @@ -67,15 +66,15 @@ public class CromwellService {
CromwellWesMapper cromwellWesMapper,
AppConfig appConfig,
CromwellConfig config,
@Qualifier("defaultAsyncOperationExecutor") ThreadPoolTaskExecutor executor
TaskExecutor defaultAsyncOperationExecutor
) {
this.client = cromwellClient;
this.pathTranslatorFactory = pathTranslatorFactory;
this.storageClient = storageClient;
this.cromwellWesMapper = cromwellWesMapper;
this.appConfig = appConfig;
this.cromwellConfig = config;
this.executor = executor;
this.defaultAsyncOperationExecutor = defaultAsyncOperationExecutor;
}


Expand Down Expand Up @@ -232,16 +231,16 @@ public RunId cancel(String runId) {
*
* @return a list of generated files for the run
*/
public RunFiles getRunFiles(String runId) throws NotFoundException {
public RunFiles getRunFiles(String runId) {
CromwellMetadataResponse metadataResponse = getMetadata(runId);
Set<String> finalFileSet = new HashSet<>();
Set<String> secondaryFileSet = new HashSet<>();
Set<String> logFileSet = new HashSet<>();
List<RunFile> files = new ArrayList<>();

Map<String, Object> outputs = metadataResponse.getOutputs();
if (!outputs.isEmpty()) {
outputs.values().forEach(output -> extractFilesFromValue(finalFileSet, output));
if (outputs != null && !outputs.isEmpty()) {
outputs.values().forEach(output -> extractFilesFromValue(finalFileSet, mapper.valueToTree(output)));
}
extractSecondaryLogFiles(secondaryFileSet, logFileSet, metadataResponse);

Expand Down Expand Up @@ -280,15 +279,17 @@ public RunFileDeletions deleteRunFiles(String runId, boolean async) {
}

public RunFileDeletion deleteRunFileAsync(RunFile runFile) {
CompletableFuture.runAsync(() -> deleteRunFile(runFile), executor);
CompletableFuture.runAsync(() -> deleteRunFile(runFile), defaultAsyncOperationExecutor);
return new RunFileDeletion(runFile, RunFileDeletion.DeletionState.ASYNC,null);
}

public RunFileDeletion deleteRunFile(RunFile runFile) {
try {
storageClient.deleteFile(runFile.getPath());
Sean-DNAstack marked this conversation as resolved.
Show resolved Hide resolved
log.info("Deleting file '{}'", runFile.getPath());
return new RunFileDeletion(runFile, RunFileDeletion.DeletionState.DELETED, null);
} catch (IOException e) {
log.error("Encountered exception while deleting file '%s': '%s'".formatted(runFile.getPath(), e.getMessage()), e);
return new RunFileDeletion(runFile, RunFileDeletion.DeletionState.FAILED, ErrorResponse.builder().errorCode(400).msg(e.getMessage()).build());
}
}
Expand Down Expand Up @@ -526,7 +527,7 @@ private JsonNode extractJsonNode(String value) throws IOException {
private void extractSecondaryLogFiles(Set<String> secondaryFileSet, Set<String> logFileSet, CromwellMetadataResponse metadataResponse){
Map<String, Object> outputs = metadataResponse.getOutputs();
if (outputs != null && !outputs.isEmpty()) {
outputs.values().forEach(output -> extractFilesFromValue(secondaryFileSet, output));
outputs.values().forEach(output -> extractFilesFromValue(secondaryFileSet, mapper.valueToTree(output)));
}
Map<String, List<CromwellTaskCall>> calls = metadataResponse.getCalls();
if (calls != null && !calls.isEmpty()) {
Expand All @@ -537,7 +538,7 @@ private void extractSecondaryLogFiles(Set<String> secondaryFileSet, Set<String>
private void extractSecondaryLogFilesFromCall(Set<String> secondaryFileSet, Set<String> logFileSet, CromwellTaskCall call){
Map<String, Object> outputs = call.getOutputs();
if (outputs != null && !outputs.isEmpty()) {
outputs.values().forEach(output -> extractFilesFromValue(secondaryFileSet, output));
outputs.values().forEach(output -> extractFilesFromValue(secondaryFileSet, mapper.valueToTree(output)));
}
String stderr = call.getStderr();
String stdout = call.getStdout();
Expand All @@ -549,26 +550,23 @@ private void extractSecondaryLogFilesFromCall(Set<String> secondaryFileSet, Set<
}
Map<String, String> backendLogs = call.getBackendLogs();
if (backendLogs != null && !backendLogs.isEmpty()) {
backendLogs.values().forEach(log -> extractFilesFromValue(logFileSet, log));
backendLogs.values().forEach(log -> extractFilesFromValue(logFileSet, mapper.valueToTree(log)));
}
CromwellMetadataResponse subWorkflowMetadata = call.getSubWorkflowMetadata();
if (subWorkflowMetadata != null) {
extractSecondaryLogFiles(secondaryFileSet,logFileSet,subWorkflowMetadata);
extractSecondaryLogFiles(secondaryFileSet, logFileSet, subWorkflowMetadata);
}
}

private void extractFilesFromValue(Set<String> fileSet, Object output) {
JsonNode node = mapper.valueToTree(output);
private void extractFilesFromValue(Set<String> fileSet, JsonNode node) {
if (node.isTextual()) {
if (storageClient.isFile(node.asText())) {
fileSet.add(node.asText());
}
} else if (node.isArray()) {
ArrayNode arrayOutput = mapper.valueToTree(output);
extractFilesFromArrayNode(fileSet, arrayOutput);
extractFilesFromArrayNode(fileSet, (ArrayNode) node);
} else if (node.isObject()) {
ObjectNode objectOutput = mapper.valueToTree(output);
extractFilesFromObjectNode(fileSet, objectOutput);
extractFilesFromObjectNode(fileSet, (ObjectNode) node);
}
}

Expand Down
Loading