diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9b79565..b9a316a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,7 +1,7 @@ image: openjdk:8 variables: - APP_VERSION: "1.0.1" + APP_VERSION: "1.0.2" APP_NAME: 'ena-file-downloader' FTP_DIR: "/nfs/ftp/pub/databases/ena/tools" diff --git a/command-line-downloader/distribution/README.txt b/command-line-downloader/distribution/README.txt index c47de55..853cf13 100644 --- a/command-line-downloader/distribution/README.txt +++ b/command-line-downloader/distribution/README.txt @@ -14,7 +14,7 @@ * limitations under the License. ******************************************************************************/ -version:1.0.1 +version:1.0.2 Ena File Downloader Copyright © EMBL 2021 | EMBL-EBI is part of the European Molecular Biology Laboratory diff --git a/command-line-downloader/src/main/java/uk/ac/ebi/ena/app/utils/MenuUtils.java b/command-line-downloader/src/main/java/uk/ac/ebi/ena/app/utils/MenuUtils.java index 302c5ac..39263f5 100644 --- a/command-line-downloader/src/main/java/uk/ac/ebi/ena/app/utils/MenuUtils.java +++ b/command-line-downloader/src/main/java/uk/ac/ebi/ena/app/utils/MenuUtils.java @@ -166,7 +166,7 @@ public static List accsFromFile(String inputValues) { } List accessionIds = Files.lines(Paths.get(inputValues), StandardCharsets.US_ASCII).collect(Collectors.toList()); - return new ArrayList<>(accessionIds.stream().map(aRow -> aRow.replace("\"", "").trim()).collect(Collectors.toSet())); + return new ArrayList<>(accessionIds.stream().map(aRow -> aRow.replace("\"", "").trim()).collect(Collectors.toCollection(LinkedHashSet::new))); } } } catch (IOException exception) { diff --git a/command-line-downloader/src/main/java/uk/ac/ebi/ena/backend/dto/EnaPortalResponse.java b/command-line-downloader/src/main/java/uk/ac/ebi/ena/backend/dto/EnaPortalResponse.java index 107c665..b373fb8 100644 --- a/command-line-downloader/src/main/java/uk/ac/ebi/ena/backend/dto/EnaPortalResponse.java +++ b/command-line-downloader/src/main/java/uk/ac/ebi/ena/backend/dto/EnaPortalResponse.java @@ -26,7 +26,8 @@ public class EnaPortalResponse { @JsonProperty("run_accession") - private String runId; + @JsonAlias("analysis_accession") + private String recordId; @JsonProperty("experiment_accession") @JsonAlias({"sample_accession", "study_accession", "analysis_accession", "run_accession"}) private String parentId; diff --git a/command-line-downloader/src/main/java/uk/ac/ebi/ena/backend/dto/FileDetail.java b/command-line-downloader/src/main/java/uk/ac/ebi/ena/backend/dto/FileDetail.java index 037b8ad..8e689d5 100644 --- a/command-line-downloader/src/main/java/uk/ac/ebi/ena/backend/dto/FileDetail.java +++ b/command-line-downloader/src/main/java/uk/ac/ebi/ena/backend/dto/FileDetail.java @@ -25,7 +25,7 @@ @AllArgsConstructor public class FileDetail { private String parentId; - private String runId; + private String recordId; private String ftpUrl; private Long bytes; private String md5; diff --git a/command-line-downloader/src/main/java/uk/ac/ebi/ena/backend/service/AccessionDetailsService.java b/command-line-downloader/src/main/java/uk/ac/ebi/ena/backend/service/AccessionDetailsService.java index aa50a2d..684b77c 100644 --- a/command-line-downloader/src/main/java/uk/ac/ebi/ena/backend/service/AccessionDetailsService.java +++ b/command-line-downloader/src/main/java/uk/ac/ebi/ena/backend/service/AccessionDetailsService.java @@ -69,7 +69,7 @@ private List createFileDetails(List enaPortalResp List bytesList = getBytes(enaPortalResponse.getBytes()); for (int i = 0; i < ftpUrlsList.size(); i++) { - fileDetails.add(new FileDetail(enaPortalResponse.getParentId(), enaPortalResponse.getRunId(), + fileDetails.add(new FileDetail(enaPortalResponse.getParentId(), enaPortalResponse.getRecordId(), ftpUrlsList.get(i), bytesList.get(i), md5List.get(i))); } } @@ -109,7 +109,7 @@ public long fetchAccessionAndDownload(DownloadFormatEnum format, String download final ExecutorService executorService = Executors.newFixedThreadPool(Constants.EXECUTOR_THREAD_COUNT); String accessionField = accessionDetailsMap.get(ACCESSION_FIELD).get(0); - String accessionType = AccessionTypeEnum.getAccessionType(accessionField).name(); + AccessionTypeEnum accessionType = AccessionTypeEnum.getAccessionType(accessionField); List accessions = accessionDetailsMap.get(ACCESSION_LIST); List> accLists = Collections.synchronizedList(Lists.partition(accessions, 10000)); long total = 0; diff --git a/command-line-downloader/src/main/java/uk/ac/ebi/ena/backend/service/EnaPortalService.java b/command-line-downloader/src/main/java/uk/ac/ebi/ena/backend/service/EnaPortalService.java index fc87fe1..aec5db4 100644 --- a/command-line-downloader/src/main/java/uk/ac/ebi/ena/backend/service/EnaPortalService.java +++ b/command-line-downloader/src/main/java/uk/ac/ebi/ena/backend/service/EnaPortalService.java @@ -38,7 +38,8 @@ import static uk.ac.ebi.ena.app.constants.Constants.ACCESSION_FIELD; /** - * This class will invoke the Portal API and fetch the {@value SEARCH_FIELDS_READ_FASTQ /SEARCH_FIELDS_SUBMITTED} for the + * This class will invoke the Portal API and fetch the {@value SEARCH_FIELDS_READ_FASTQ /SEARCH_FIELDS_SUBMITTED} for + * the * accessionIdList and dataType */ @Component @@ -58,10 +59,12 @@ public class EnaPortalService { public static final String FASTQ_ASPERA_FIELD = ",fastq_aspera"; public static final String SUBMITTED_ASPERA_FIELD = ",submitted_aspera"; - private static final String PORTAL_API_READ_RUN_SEARCH_URL = "https://www.ebi.ac.uk/ena/portal/api/search?result=read_run" + - "&includeAccessionType=%s&fields=%s&format=json"; - private static final String PORTAL_API_ANALYSIS_SEARCH_URL = "https://www.ebi.ac.uk/ena/portal/api/search?result=analysis" + - "&includeAccessionType=%s&fields=%s&format=json"; + private static final String PORTAL_API_READ_RUN_SEARCH_URL = "https://www.ebi.ac.uk/ena/portal/api/search?result" + + "=read_run" + + "&includeAccessionType=%s&fields=%s&format=json&limit=0"; + private static final String PORTAL_API_ANALYSIS_SEARCH_URL = "https://www.ebi.ac.uk/ena/portal/api/search?result" + + "=analysis" + + "&includeAccessionType=%s&fields=%s&format=json&limit=0"; private static final String PORTAL_API_SUPPORT_URL = "https://www.ebi.ac.uk/ena/portal/api/support?email=datasubs" + "@ebi.ac.uk&message=%s&to=%s&subject=%s&name=%s"; @@ -79,7 +82,8 @@ public class EnaPortalService { private final RestTemplate restTemplate; /** - * This API will invoke the Portal API and fetch the {@value SEARCH_FIELDS_READ_FASTQ /SEARCH_FIELDS_SUBMITTED} for the + * This API will invoke the Portal API and fetch the {@value SEARCH_FIELDS_READ_FASTQ /SEARCH_FIELDS_SUBMITTED} + * for the * accessionIdList and dataType * * @param accessionIdList The experimentIds @@ -89,7 +93,8 @@ public class EnaPortalService { * @return The details for the accession Ids */ public List getPortalResponses(List accessionIdList, DownloadFormatEnum format, - ProtocolEnum protocol, Map> accessionDetailsMap) { + ProtocolEnum protocol, + Map> accessionDetailsMap) { String accessionField = accessionDetailsMap.get(ACCESSION_FIELD).get(0); String accessionType = AccessionTypeEnum.getAccessionType(accessionField).name().toLowerCase(); @@ -102,58 +107,86 @@ public List getPortalResponses(List accessionIdList, case SAMPLE: switch (format) { case READS_FASTQ: - portalAPIEndpoint = String.format(PORTAL_API_READ_RUN_SEARCH_URL, AccessionTypeEnum.SAMPLE.name().toLowerCase(), AccessionTypeEnum.SAMPLE.getAccessionField() + SEARCH_FIELDS_READ_FASTQ + FASTQ_FTP_FIELD); + portalAPIEndpoint = String.format(PORTAL_API_READ_RUN_SEARCH_URL, + AccessionTypeEnum.SAMPLE.name().toLowerCase(), + AccessionTypeEnum.SAMPLE.getAccessionField() + SEARCH_FIELDS_READ_FASTQ + FASTQ_FTP_FIELD); break outer; case READS_SUBMITTED: - portalAPIEndpoint = String.format(PORTAL_API_READ_RUN_SEARCH_URL, AccessionTypeEnum.SAMPLE.name().toLowerCase(), AccessionTypeEnum.SAMPLE.getAccessionField() + SEARCH_FIELDS_SUBMITTED + SUBMITTED_FTP_FIELD); + portalAPIEndpoint = String.format(PORTAL_API_READ_RUN_SEARCH_URL, + AccessionTypeEnum.SAMPLE.name().toLowerCase(), + AccessionTypeEnum.SAMPLE.getAccessionField() + SEARCH_FIELDS_SUBMITTED + SUBMITTED_FTP_FIELD); break outer; case ANALYSIS_SUBMITTED: - portalAPIEndpoint = String.format(PORTAL_API_ANALYSIS_SEARCH_URL, AccessionTypeEnum.ANALYSIS.name().toLowerCase(), AccessionTypeEnum.SAMPLE.getAccessionField() + SEARCH_FIELDS_SUBMITTED + SUBMITTED_FTP_FIELD); + portalAPIEndpoint = String.format(PORTAL_API_ANALYSIS_SEARCH_URL, + AccessionTypeEnum.ANALYSIS.name().toLowerCase(), + AccessionTypeEnum.SAMPLE.getAccessionField() + SEARCH_FIELDS_SUBMITTED + SUBMITTED_FTP_FIELD); break outer; case ANALYSIS_GENERATED: - portalAPIEndpoint = String.format(PORTAL_API_ANALYSIS_SEARCH_URL, AccessionTypeEnum.ANALYSIS.name().toLowerCase(), AccessionTypeEnum.SAMPLE.getAccessionField() + SEARCH_FIELDS_GENERATED + GENERATED_FTP_FIELD); + portalAPIEndpoint = String.format(PORTAL_API_ANALYSIS_SEARCH_URL, + AccessionTypeEnum.ANALYSIS.name().toLowerCase(), + AccessionTypeEnum.SAMPLE.getAccessionField() + SEARCH_FIELDS_GENERATED + GENERATED_FTP_FIELD); break outer; } case STUDY: switch (format) { case READS_FASTQ: - portalAPIEndpoint = String.format(PORTAL_API_READ_RUN_SEARCH_URL, AccessionTypeEnum.STUDY.name().toLowerCase(), AccessionTypeEnum.STUDY.getAccessionField() + SEARCH_FIELDS_READ_FASTQ + FASTQ_FTP_FIELD); + portalAPIEndpoint = String.format(PORTAL_API_READ_RUN_SEARCH_URL, + AccessionTypeEnum.STUDY.name().toLowerCase(), + AccessionTypeEnum.STUDY.getAccessionField() + SEARCH_FIELDS_READ_FASTQ + FASTQ_FTP_FIELD); break outer; case READS_SUBMITTED: - portalAPIEndpoint = String.format(PORTAL_API_READ_RUN_SEARCH_URL, AccessionTypeEnum.STUDY.name().toLowerCase(), AccessionTypeEnum.STUDY.getAccessionField() + SEARCH_FIELDS_SUBMITTED + SUBMITTED_FTP_FIELD); + portalAPIEndpoint = String.format(PORTAL_API_READ_RUN_SEARCH_URL, + AccessionTypeEnum.STUDY.name().toLowerCase(), + AccessionTypeEnum.STUDY.getAccessionField() + SEARCH_FIELDS_SUBMITTED + SUBMITTED_FTP_FIELD); break outer; case ANALYSIS_SUBMITTED: - portalAPIEndpoint = String.format(PORTAL_API_ANALYSIS_SEARCH_URL, AccessionTypeEnum.STUDY.name().toLowerCase(), AccessionTypeEnum.STUDY.getAccessionField() + SEARCH_FIELDS_SUBMITTED + SUBMITTED_FTP_FIELD); + portalAPIEndpoint = String.format(PORTAL_API_ANALYSIS_SEARCH_URL, + AccessionTypeEnum.STUDY.name().toLowerCase(), + AccessionTypeEnum.STUDY.getAccessionField() + SEARCH_FIELDS_SUBMITTED + SUBMITTED_FTP_FIELD); break outer; case ANALYSIS_GENERATED: - portalAPIEndpoint = String.format(PORTAL_API_ANALYSIS_SEARCH_URL, AccessionTypeEnum.STUDY.name().toLowerCase(), AccessionTypeEnum.STUDY.getAccessionField() + SEARCH_FIELDS_GENERATED + GENERATED_FTP_FIELD); + portalAPIEndpoint = String.format(PORTAL_API_ANALYSIS_SEARCH_URL, + AccessionTypeEnum.STUDY.name().toLowerCase(), + AccessionTypeEnum.STUDY.getAccessionField() + SEARCH_FIELDS_GENERATED + GENERATED_FTP_FIELD); break outer; } case EXPERIMENT: switch (format) { case READS_FASTQ: - portalAPIEndpoint = String.format(PORTAL_API_READ_RUN_SEARCH_URL, AccessionTypeEnum.EXPERIMENT.name().toLowerCase(), AccessionTypeEnum.EXPERIMENT.getAccessionField() + SEARCH_FIELDS_READ_FASTQ + FASTQ_FTP_FIELD); + portalAPIEndpoint = String.format(PORTAL_API_READ_RUN_SEARCH_URL, + AccessionTypeEnum.EXPERIMENT.name().toLowerCase(), + AccessionTypeEnum.EXPERIMENT.getAccessionField() + SEARCH_FIELDS_READ_FASTQ + FASTQ_FTP_FIELD); break outer; case READS_SUBMITTED: - portalAPIEndpoint = String.format(PORTAL_API_READ_RUN_SEARCH_URL, AccessionTypeEnum.EXPERIMENT.name().toLowerCase(), AccessionTypeEnum.EXPERIMENT.getAccessionField() + SEARCH_FIELDS_SUBMITTED + SUBMITTED_FTP_FIELD); + portalAPIEndpoint = String.format(PORTAL_API_READ_RUN_SEARCH_URL, + AccessionTypeEnum.EXPERIMENT.name().toLowerCase(), + AccessionTypeEnum.EXPERIMENT.getAccessionField() + SEARCH_FIELDS_SUBMITTED + SUBMITTED_FTP_FIELD); break outer; } case RUN: switch (format) { case READS_FASTQ: - portalAPIEndpoint = String.format(PORTAL_API_READ_RUN_SEARCH_URL, AccessionTypeEnum.RUN.name().toLowerCase(), AccessionTypeEnum.RUN.getAccessionField() + SEARCH_FIELDS_READ_FASTQ + FASTQ_FTP_FIELD); + portalAPIEndpoint = String.format(PORTAL_API_READ_RUN_SEARCH_URL, + AccessionTypeEnum.RUN.name().toLowerCase(), + AccessionTypeEnum.RUN.getAccessionField() + SEARCH_FIELDS_READ_FASTQ + FASTQ_FTP_FIELD); break outer; case READS_SUBMITTED: - portalAPIEndpoint = String.format(PORTAL_API_READ_RUN_SEARCH_URL, AccessionTypeEnum.RUN.name().toLowerCase(), AccessionTypeEnum.RUN.getAccessionField() + SEARCH_FIELDS_SUBMITTED + SUBMITTED_FTP_FIELD); + portalAPIEndpoint = String.format(PORTAL_API_READ_RUN_SEARCH_URL, + AccessionTypeEnum.RUN.name().toLowerCase(), + AccessionTypeEnum.RUN.getAccessionField() + SEARCH_FIELDS_SUBMITTED + SUBMITTED_FTP_FIELD); break outer; } case ANALYSIS: switch (format) { case ANALYSIS_SUBMITTED: - portalAPIEndpoint = String.format(PORTAL_API_ANALYSIS_SEARCH_URL, AccessionTypeEnum.ANALYSIS.name().toLowerCase(), AccessionTypeEnum.ANALYSIS.getAccessionField() + SEARCH_FIELDS_SUBMITTED + SUBMITTED_FTP_FIELD); + portalAPIEndpoint = String.format(PORTAL_API_ANALYSIS_SEARCH_URL, + AccessionTypeEnum.ANALYSIS.name().toLowerCase(), + AccessionTypeEnum.ANALYSIS.getAccessionField() + SEARCH_FIELDS_SUBMITTED + SUBMITTED_FTP_FIELD); break outer; case ANALYSIS_GENERATED: - portalAPIEndpoint = String.format(PORTAL_API_ANALYSIS_SEARCH_URL, AccessionTypeEnum.ANALYSIS.name().toLowerCase(), AccessionTypeEnum.ANALYSIS.getAccessionField() + SEARCH_FIELDS_GENERATED + GENERATED_FTP_FIELD); + portalAPIEndpoint = String.format(PORTAL_API_ANALYSIS_SEARCH_URL, + AccessionTypeEnum.ANALYSIS.name().toLowerCase(), + AccessionTypeEnum.ANALYSIS.getAccessionField() + SEARCH_FIELDS_GENERATED + GENERATED_FTP_FIELD); break outer; } } @@ -162,58 +195,86 @@ public List getPortalResponses(List accessionIdList, case SAMPLE: switch (format) { case READS_FASTQ: - portalAPIEndpoint = String.format(PORTAL_API_READ_RUN_SEARCH_URL, AccessionTypeEnum.SAMPLE.name().toLowerCase(), AccessionTypeEnum.SAMPLE.getAccessionField() + SEARCH_FIELDS_READ_FASTQ + FASTQ_ASPERA_FIELD); + portalAPIEndpoint = String.format(PORTAL_API_READ_RUN_SEARCH_URL, + AccessionTypeEnum.SAMPLE.name().toLowerCase(), + AccessionTypeEnum.SAMPLE.getAccessionField() + SEARCH_FIELDS_READ_FASTQ + FASTQ_ASPERA_FIELD); break outer; case READS_SUBMITTED: - portalAPIEndpoint = String.format(PORTAL_API_READ_RUN_SEARCH_URL, AccessionTypeEnum.SAMPLE.name().toLowerCase(), AccessionTypeEnum.SAMPLE.getAccessionField() + SEARCH_FIELDS_SUBMITTED + SUBMITTED_ASPERA_FIELD); + portalAPIEndpoint = String.format(PORTAL_API_READ_RUN_SEARCH_URL, + AccessionTypeEnum.SAMPLE.name().toLowerCase(), + AccessionTypeEnum.SAMPLE.getAccessionField() + SEARCH_FIELDS_SUBMITTED + SUBMITTED_ASPERA_FIELD); break outer; case ANALYSIS_SUBMITTED: - portalAPIEndpoint = String.format(PORTAL_API_ANALYSIS_SEARCH_URL, AccessionTypeEnum.SAMPLE.name().toLowerCase(), AccessionTypeEnum.SAMPLE.getAccessionField() + SEARCH_FIELDS_SUBMITTED + SUBMITTED_ASPERA_FIELD); + portalAPIEndpoint = String.format(PORTAL_API_ANALYSIS_SEARCH_URL, + AccessionTypeEnum.SAMPLE.name().toLowerCase(), + AccessionTypeEnum.SAMPLE.getAccessionField() + SEARCH_FIELDS_SUBMITTED + SUBMITTED_ASPERA_FIELD); break outer; case ANALYSIS_GENERATED: - portalAPIEndpoint = String.format(PORTAL_API_ANALYSIS_SEARCH_URL, AccessionTypeEnum.SAMPLE.name().toLowerCase(), AccessionTypeEnum.SAMPLE.getAccessionField() + SEARCH_FIELDS_GENERATED + GENERATED_ASPERA_FIELD); + portalAPIEndpoint = String.format(PORTAL_API_ANALYSIS_SEARCH_URL, + AccessionTypeEnum.SAMPLE.name().toLowerCase(), + AccessionTypeEnum.SAMPLE.getAccessionField() + SEARCH_FIELDS_GENERATED + GENERATED_ASPERA_FIELD); break outer; } case STUDY: switch (format) { case READS_FASTQ: - portalAPIEndpoint = String.format(PORTAL_API_READ_RUN_SEARCH_URL, AccessionTypeEnum.STUDY.name().toLowerCase(), AccessionTypeEnum.STUDY.getAccessionField() + SEARCH_FIELDS_READ_FASTQ + FASTQ_ASPERA_FIELD); + portalAPIEndpoint = String.format(PORTAL_API_READ_RUN_SEARCH_URL, + AccessionTypeEnum.STUDY.name().toLowerCase(), + AccessionTypeEnum.STUDY.getAccessionField() + SEARCH_FIELDS_READ_FASTQ + FASTQ_ASPERA_FIELD); break outer; case READS_SUBMITTED: - portalAPIEndpoint = String.format(PORTAL_API_READ_RUN_SEARCH_URL, AccessionTypeEnum.STUDY.name().toLowerCase(), AccessionTypeEnum.STUDY.getAccessionField() + SEARCH_FIELDS_SUBMITTED + SUBMITTED_ASPERA_FIELD); + portalAPIEndpoint = String.format(PORTAL_API_READ_RUN_SEARCH_URL, + AccessionTypeEnum.STUDY.name().toLowerCase(), + AccessionTypeEnum.STUDY.getAccessionField() + SEARCH_FIELDS_SUBMITTED + SUBMITTED_ASPERA_FIELD); break outer; case ANALYSIS_SUBMITTED: - portalAPIEndpoint = String.format(PORTAL_API_ANALYSIS_SEARCH_URL, AccessionTypeEnum.STUDY.name().toLowerCase(), AccessionTypeEnum.STUDY.getAccessionField() + SEARCH_FIELDS_SUBMITTED + SUBMITTED_ASPERA_FIELD); + portalAPIEndpoint = String.format(PORTAL_API_ANALYSIS_SEARCH_URL, + AccessionTypeEnum.STUDY.name().toLowerCase(), + AccessionTypeEnum.STUDY.getAccessionField() + SEARCH_FIELDS_SUBMITTED + SUBMITTED_ASPERA_FIELD); break outer; case ANALYSIS_GENERATED: - portalAPIEndpoint = String.format(PORTAL_API_ANALYSIS_SEARCH_URL, AccessionTypeEnum.STUDY.name().toLowerCase(), AccessionTypeEnum.STUDY.getAccessionField() + SEARCH_FIELDS_GENERATED + GENERATED_ASPERA_FIELD); + portalAPIEndpoint = String.format(PORTAL_API_ANALYSIS_SEARCH_URL, + AccessionTypeEnum.STUDY.name().toLowerCase(), + AccessionTypeEnum.STUDY.getAccessionField() + SEARCH_FIELDS_GENERATED + GENERATED_ASPERA_FIELD); break outer; } case EXPERIMENT: switch (format) { case READS_FASTQ: - portalAPIEndpoint = String.format(PORTAL_API_READ_RUN_SEARCH_URL, AccessionTypeEnum.EXPERIMENT.name().toLowerCase(), AccessionTypeEnum.EXPERIMENT.getAccessionField() + SEARCH_FIELDS_READ_FASTQ + FASTQ_ASPERA_FIELD); + portalAPIEndpoint = String.format(PORTAL_API_READ_RUN_SEARCH_URL, + AccessionTypeEnum.EXPERIMENT.name().toLowerCase(), + AccessionTypeEnum.EXPERIMENT.getAccessionField() + SEARCH_FIELDS_READ_FASTQ + FASTQ_ASPERA_FIELD); break outer; case READS_SUBMITTED: - portalAPIEndpoint = String.format(PORTAL_API_READ_RUN_SEARCH_URL, AccessionTypeEnum.EXPERIMENT.name().toLowerCase(), AccessionTypeEnum.EXPERIMENT.getAccessionField() + SEARCH_FIELDS_SUBMITTED + SUBMITTED_ASPERA_FIELD); + portalAPIEndpoint = String.format(PORTAL_API_READ_RUN_SEARCH_URL, + AccessionTypeEnum.EXPERIMENT.name().toLowerCase(), + AccessionTypeEnum.EXPERIMENT.getAccessionField() + SEARCH_FIELDS_SUBMITTED + SUBMITTED_ASPERA_FIELD); break outer; } case RUN: switch (format) { case READS_FASTQ: - portalAPIEndpoint = String.format(PORTAL_API_READ_RUN_SEARCH_URL, AccessionTypeEnum.RUN.name().toLowerCase(), AccessionTypeEnum.RUN.getAccessionField() + SEARCH_FIELDS_READ_FASTQ + FASTQ_ASPERA_FIELD); + portalAPIEndpoint = String.format(PORTAL_API_READ_RUN_SEARCH_URL, + AccessionTypeEnum.RUN.name().toLowerCase(), + AccessionTypeEnum.RUN.getAccessionField() + SEARCH_FIELDS_READ_FASTQ + FASTQ_ASPERA_FIELD); break outer; case READS_SUBMITTED: - portalAPIEndpoint = String.format(PORTAL_API_READ_RUN_SEARCH_URL, AccessionTypeEnum.RUN.name().toLowerCase(), AccessionTypeEnum.RUN.getAccessionField() + SEARCH_FIELDS_SUBMITTED + SUBMITTED_ASPERA_FIELD); + portalAPIEndpoint = String.format(PORTAL_API_READ_RUN_SEARCH_URL, + AccessionTypeEnum.RUN.name().toLowerCase(), + AccessionTypeEnum.RUN.getAccessionField() + SEARCH_FIELDS_SUBMITTED + SUBMITTED_ASPERA_FIELD); break outer; } case ANALYSIS: switch (format) { case ANALYSIS_SUBMITTED: - portalAPIEndpoint = String.format(PORTAL_API_ANALYSIS_SEARCH_URL, AccessionTypeEnum.ANALYSIS.name().toLowerCase(), AccessionTypeEnum.ANALYSIS.getAccessionField() + SEARCH_FIELDS_SUBMITTED + SUBMITTED_ASPERA_FIELD); + portalAPIEndpoint = String.format(PORTAL_API_ANALYSIS_SEARCH_URL, + AccessionTypeEnum.ANALYSIS.name().toLowerCase(), + AccessionTypeEnum.ANALYSIS.getAccessionField() + SEARCH_FIELDS_SUBMITTED + SUBMITTED_ASPERA_FIELD); break outer; case ANALYSIS_GENERATED: - portalAPIEndpoint = String.format(PORTAL_API_ANALYSIS_SEARCH_URL, AccessionTypeEnum.ANALYSIS.name().toLowerCase(), AccessionTypeEnum.ANALYSIS.getAccessionField() + SEARCH_FIELDS_GENERATED + GENERATED_ASPERA_FIELD); + portalAPIEndpoint = String.format(PORTAL_API_ANALYSIS_SEARCH_URL, + AccessionTypeEnum.ANALYSIS.name().toLowerCase(), + AccessionTypeEnum.ANALYSIS.getAccessionField() + SEARCH_FIELDS_GENERATED + GENERATED_ASPERA_FIELD); break outer; } } diff --git a/command-line-downloader/src/main/java/uk/ac/ebi/ena/backend/service/FileDownloaderClient.java b/command-line-downloader/src/main/java/uk/ac/ebi/ena/backend/service/FileDownloaderClient.java index 26815d0..e14b928 100644 --- a/command-line-downloader/src/main/java/uk/ac/ebi/ena/backend/service/FileDownloaderClient.java +++ b/command-line-downloader/src/main/java/uk/ac/ebi/ena/backend/service/FileDownloaderClient.java @@ -36,6 +36,7 @@ import org.apache.http.impl.client.HttpClientBuilder; import org.springframework.stereotype.Component; import uk.ac.ebi.ena.app.constants.Constants; +import uk.ac.ebi.ena.app.menu.enums.AccessionTypeEnum; import uk.ac.ebi.ena.app.menu.enums.DownloadFormatEnum; import uk.ac.ebi.ena.app.utils.CommonUtils; import uk.ac.ebi.ena.backend.dto.FileDetail; @@ -197,7 +198,7 @@ private List getAsperaCommandParts(String asperaLocation, FileDetail fil public Future startDownloadAspera(ExecutorService executorService, List fileDetails, String asperaLocation, String downloadLocation, - String accessionType, DownloadFormatEnum format, int set) { + AccessionTypeEnum accessionType, DownloadFormatEnum format, int set) { FileDownloadStatus fileDownloadStatus = new FileDownloadStatus(fileDetails.size(), 0, new ArrayList<>()); return executorService.submit(() -> { diff --git a/command-line-downloader/src/main/java/uk/ac/ebi/ena/backend/service/FileDownloaderService.java b/command-line-downloader/src/main/java/uk/ac/ebi/ena/backend/service/FileDownloaderService.java index e97ac69..5060761 100644 --- a/command-line-downloader/src/main/java/uk/ac/ebi/ena/backend/service/FileDownloaderService.java +++ b/command-line-downloader/src/main/java/uk/ac/ebi/ena/backend/service/FileDownloaderService.java @@ -28,6 +28,7 @@ import org.springframework.util.Assert; import org.springframework.util.DigestUtils; import uk.ac.ebi.ena.app.constants.Constants; +import uk.ac.ebi.ena.app.menu.enums.AccessionTypeEnum; import uk.ac.ebi.ena.app.menu.enums.DownloadFormatEnum; import uk.ac.ebi.ena.backend.dto.FileDetail; import uk.ac.ebi.ena.backend.enums.FileDownloadStatus; @@ -95,18 +96,19 @@ public static ProgressBarBuilder getBProgressBar(String s, long size) { .setUnit("B", 1); // setting the progress bar to use MB as the unit } - public static String getFileDownloadPath(String downloadLoc, String accessionType, DownloadFormatEnum format, + public static String getFileDownloadPath(String downloadLoc, AccessionTypeEnum accessionType, + DownloadFormatEnum format, FileDetail fileDetail) { switch (accessionType) { - case "RUN": + case RUN: return downloadLoc + File.separator - + StringUtils.lowerCase(format.toString()) + File.separator + fileDetail.getRunId(); - case "PROJECT": - case "EXPERIMENT": - case "SAMPLE": + + StringUtils.lowerCase(format.toString()) + File.separator + fileDetail.getRecordId(); + case STUDY: + case EXPERIMENT: + case SAMPLE: return downloadLoc + File.separator - + StringUtils.lowerCase(format.toString()) + File.separator + fileDetail.getParentId() + File.separator + fileDetail.getRunId(); - case "ANALYSIS": + + StringUtils.lowerCase(format.toString()) + File.separator + fileDetail.getParentId() + File.separator + fileDetail.getRecordId(); + case ANALYSIS: return downloadLoc + File.separator + StringUtils.lowerCase(format.toString()) + File.separator + fileDetail.getParentId(); } @@ -144,7 +146,7 @@ private void deleteIfPartialFileExists(Path partialFilePath, String partialFileN public Future startDownload(ExecutorService executorService, List fileDetails, - String downloadLoc, String accessionType, + String downloadLoc, AccessionTypeEnum accessionType, DownloadFormatEnum format, int set) { FileDownloadStatus fileDownloadStatus = new FileDownloadStatus(fileDetails.size(), 0, new ArrayList<>()); @@ -205,6 +207,7 @@ public Future startDownload(ExecutorService executorService, fileDownloaderPath + File.separator + remoteFileName, bytesCopied); if (isDownloaded) { fileProgressBar.stepBy(1); + log.info("{} completed.", fileDownloaderPath + File.separator + remoteFileName); fileDownloadStatus.setSuccesssful(fileDownloadStatus.getSuccesssful() + 1); } else { log.error("Failed to download file:{}, experimentId:{}", remoteFileName, diff --git a/command-line-downloader/src/test/java/uk/ac/ebi/ena/backend/AccessionDetailsServiceTest.java b/command-line-downloader/src/test/java/uk/ac/ebi/ena/backend/AccessionDetailsServiceTest.java index a7ce60c..ca53ef7 100644 --- a/command-line-downloader/src/test/java/uk/ac/ebi/ena/backend/AccessionDetailsServiceTest.java +++ b/command-line-downloader/src/test/java/uk/ac/ebi/ena/backend/AccessionDetailsServiceTest.java @@ -8,6 +8,7 @@ import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; import org.mockito.junit.jupiter.MockitoExtension; +import uk.ac.ebi.ena.app.menu.enums.AccessionTypeEnum; import uk.ac.ebi.ena.app.menu.enums.DownloadFormatEnum; import uk.ac.ebi.ena.app.menu.enums.ProtocolEnum; import uk.ac.ebi.ena.app.utils.CommonUtils; @@ -63,7 +64,7 @@ public void testFetchAccessionAndDownloadWhenSuccess() throws ExecutionException final Future mockedFuture = Mockito.mock(Future.class); when(mockedFuture.get()).thenReturn(new FileDownloadStatus(0, 0, new ArrayList<>())); Mockito.when(fileDownloaderService.startDownload(Mockito.any(ExecutorService.class), Mockito.any(List.class), - Mockito.any(String.class), Mockito.any(String.class), Mockito.any(DownloadFormatEnum.class), + Mockito.any(String.class), Mockito.any(AccessionTypeEnum.class), Mockito.any(DownloadFormatEnum.class), Mockito.anyInt())).thenReturn(mockedFuture); //ACT accessionDetailsService.fetchAccessionAndDownload(format, downloadLocation, accessionDetailsMap, protocol, asperaLocation, recipientEmailId); @@ -72,7 +73,7 @@ public void testFetchAccessionAndDownloadWhenSuccess() throws ExecutionException Mockito.anyMap()); verify(fileDownloaderService, times(3)).startDownload(Mockito.any(ExecutorService.class), Mockito.anyList(), Mockito.any(String.class), - Mockito.any(String.class), Mockito.any(DownloadFormatEnum.class), Mockito.anyInt()); + Mockito.any(AccessionTypeEnum.class), Mockito.any(DownloadFormatEnum.class), Mockito.anyInt()); } @@ -82,21 +83,21 @@ private List getPortalResponses() { List portalResponses = new ArrayList<>(); EnaPortalResponse enaPortalResponse1 = new EnaPortalResponse(); enaPortalResponse1.setParentId("SRX2000905"); - enaPortalResponse1.setRunId("SRR4000583"); + enaPortalResponse1.setRecordId("SRR4000583"); enaPortalResponse1.setBytes("1174738707"); enaPortalResponse1.setUrl("ftp.sra.ebi.ac.uk/vol1/fastq/SRR400/003/SRR4000583/SRR4000583.fastq.gz"); enaPortalResponse1.setMd5("a991ce890047ffca760c6de2617b5fec"); portalResponses.add(enaPortalResponse1); EnaPortalResponse enaPortalResponse2 = new EnaPortalResponse(); enaPortalResponse2.setParentId("SRX6415696"); - enaPortalResponse2.setRunId("SRR9654360"); + enaPortalResponse2.setRecordId("SRR9654360"); enaPortalResponse2.setBytes("14139836"); enaPortalResponse2.setUrl("ftp.sra.ebi.ac.uk/vol1/fastq/SRR965/000/SRR9654360/SRR9654360.fastq.gz"); enaPortalResponse2.setMd5("f3611f35a977b8b82a7adcf0a28c397d"); portalResponses.add(enaPortalResponse2); EnaPortalResponse enaPortalResponse3 = new EnaPortalResponse(); enaPortalResponse3.setParentId("SRX6415695"); - enaPortalResponse3.setRunId("SRR9654361"); + enaPortalResponse3.setRecordId("SRR9654361"); enaPortalResponse3.setBytes("15541843"); enaPortalResponse3.setUrl("ftp.sra.ebi.ac.uk/vol1/fastq/SRR965/001/SRR9654361/SRR9654361.fastq.gz"); enaPortalResponse3.setMd5("1236b79cd93a63289841765aabacb880"); diff --git a/command-line-downloader/src/test/java/uk/ac/ebi/ena/backend/EnaPortalServiceTest.java b/command-line-downloader/src/test/java/uk/ac/ebi/ena/backend/EnaPortalServiceTest.java index 8b2d162..c14ad51 100644 --- a/command-line-downloader/src/test/java/uk/ac/ebi/ena/backend/EnaPortalServiceTest.java +++ b/command-line-downloader/src/test/java/uk/ac/ebi/ena/backend/EnaPortalServiceTest.java @@ -72,21 +72,21 @@ private EnaPortalResponse[] getPortalResponses() { List portalResponses = new ArrayList<>(); EnaPortalResponse enaPortalResponse1 = new EnaPortalResponse(); enaPortalResponse1.setParentId("SRX2000905"); - enaPortalResponse1.setRunId("SRR4000583"); + enaPortalResponse1.setRecordId("SRR4000583"); enaPortalResponse1.setBytes("1174738707"); enaPortalResponse1.setUrl("ftp.sra.ebi.ac.uk/vol1/fastq/SRR400/003/SRR4000583/SRR4000583.fastq.gz"); enaPortalResponse1.setMd5("a991ce890047ffca760c6de2617b5fec"); portalResponses.add(enaPortalResponse1); EnaPortalResponse enaPortalResponse2 = new EnaPortalResponse(); enaPortalResponse2.setParentId("SRX6415696"); - enaPortalResponse2.setRunId("SRR9654360"); + enaPortalResponse2.setRecordId("SRR9654360"); enaPortalResponse2.setBytes("14139836"); enaPortalResponse2.setUrl("ftp.sra.ebi.ac.uk/vol1/fastq/SRR965/000/SRR9654360/SRR9654360.fastq.gz"); enaPortalResponse2.setMd5("f3611f35a977b8b82a7adcf0a28c397d"); portalResponses.add(enaPortalResponse2); EnaPortalResponse enaPortalResponse3 = new EnaPortalResponse(); enaPortalResponse3.setParentId("SRX6415695"); - enaPortalResponse3.setRunId("SRR9654361"); + enaPortalResponse3.setRecordId("SRR9654361"); enaPortalResponse3.setBytes("15541843"); enaPortalResponse3.setUrl("ftp.sra.ebi.ac.uk/vol1/fastq/SRR965/001/SRR9654361/SRR9654361.fastq.gz"); enaPortalResponse3.setMd5("1236b79cd93a63289841765aabacb880"); diff --git a/command-line-downloader/src/test/java/uk/ac/ebi/ena/backend/FileDownloaderServiceTest.java b/command-line-downloader/src/test/java/uk/ac/ebi/ena/backend/FileDownloaderServiceTest.java index 2954cba..d58cc9a 100644 --- a/command-line-downloader/src/test/java/uk/ac/ebi/ena/backend/FileDownloaderServiceTest.java +++ b/command-line-downloader/src/test/java/uk/ac/ebi/ena/backend/FileDownloaderServiceTest.java @@ -8,6 +8,7 @@ import org.mockito.InjectMocks; import org.mockito.junit.MockitoJUnitRunner; import org.mockito.junit.jupiter.MockitoExtension; +import uk.ac.ebi.ena.app.menu.enums.AccessionTypeEnum; import uk.ac.ebi.ena.app.menu.enums.DownloadFormatEnum; import uk.ac.ebi.ena.backend.dto.FileDetail; import uk.ac.ebi.ena.backend.enums.FileDownloadStatus; @@ -40,12 +41,12 @@ public void testStartDownload_UsingFtp() throws ExecutionException, InterruptedE List fileDetailList = new ArrayList<>(); FileDetail fileDetail = createFileDetailFtp(); fileDetailList.add(fileDetail); - String accessionType = "EXPERIMENT"; + DownloadFormatEnum format = DownloadFormatEnum.READS_FASTQ; int set = 1; //ACT FileDownloadStatus fileDownloadStatus = fileDownloaderService. - startDownload(executorService, fileDetailList, downloadFolderPath, accessionType, format, set).get(); + startDownload(executorService, fileDetailList, downloadFolderPath, AccessionTypeEnum.EXPERIMENT, format, set).get(); System.out.println(fileDownloadStatus); } @@ -59,12 +60,11 @@ public void testStartDownload_UsingAspera() throws ExecutionException, Interrupt FileDetail fileDetail = createFileDetailAspera(); fileDetailList.add(fileDetail); String asperaLocation = "C:\\Users\\suman\\AppData\\Local\\Programs\\Aspera\\Aspera Connect\\";//local aspera connect folder - String accessionType = "EXPERIMENT"; DownloadFormatEnum format = DownloadFormatEnum.READS_FASTQ; int set = 1; //ACT FileDownloadStatus fileDownloadStatus = fileDownloaderClient.startDownloadAspera - (executorService, fileDetailList, asperaLocation, downloadFolderPath, accessionType, format, set).get(); + (executorService, fileDetailList, asperaLocation, downloadFolderPath, AccessionTypeEnum.EXPERIMENT, format, set).get(); Assert.assertEquals(1, fileDownloadStatus.getSuccesssful()); }