From 4db0ee70c3c40135e9a2e3931c808448dc5650a6 Mon Sep 17 00:00:00 2001 From: suranj Date: Fri, 25 Aug 2017 16:37:42 +0100 Subject: [PATCH] v1.1.1. Handling for illegal chars in file name. portal api for acc search. fallback for other result type. --- README.md | 6 ++--- .../ebi/ena/downloader/gui/DownloadTask.java | 25 ++++-------------- .../ena/downloader/gui/ResultsController.java | 1 - .../downloader/gui/custom/MD5TableCell.java | 9 ++++--- .../downloader/service/DownloadService.java | 3 ++- .../downloader/service/WarehouseQuery.java | 17 +++++------- .../service/aspera/AsperaUtility.java | 17 ++++++------ .../service/ftp/CommonsFTPUtility.java | 26 ++----------------- .../downloader/service/ftp/FTP4JUtility.java | 7 ++--- 9 files changed, 37 insertions(+), 74 deletions(-) diff --git a/README.md b/README.md index 6275dba..e8f33df 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# ENA File Downloader version 1.1 +# ENA File Downloader version 1.1.1 Graphical user interface (GUI) for bulk downloading of run/analysis files from ENA via FTP or Aspera. @@ -9,9 +9,9 @@ Copyright 2017 EMBL - European Bioinformatics Institute Licensed under the Apach You may obtain a copy of the License at: http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. -# Installation +# Download -Download the latest release (.zip file) from the Releases section and extract it to a location of your choice. The archive contains an executable jar which contains all dependencies. +Download the latest release (.zip file) from the Releases section of [the GitHub project](https://github.com/enasequence/ena-ftp-downloader) and extract it to a location of your choice. The archive contains an executable jar which contains all dependencies. # Dependencies diff --git a/src/main/java/uk/ac/ebi/ena/downloader/gui/DownloadTask.java b/src/main/java/uk/ac/ebi/ena/downloader/gui/DownloadTask.java index d0b2235..23735a2 100644 --- a/src/main/java/uk/ac/ebi/ena/downloader/gui/DownloadTask.java +++ b/src/main/java/uk/ac/ebi/ena/downloader/gui/DownloadTask.java @@ -15,7 +15,7 @@ * Created by suranj on 01/08/2016. */ public class DownloadTask extends Task { - private final static int RETRY_COUNT = 10; + private final static int RETRY_COUNT = 5; private final static Logger log = LoggerFactory.getLogger(DownloadTask.class); private final RemoteFile file; @@ -58,12 +58,7 @@ protected Void call() throws Exception { log.debug("calling success:" + file.getName()); file.setDownloaded(true); succeeded(); - /*Platform.runLater(new Runnable() { - @Override - public void run() { - } - });*/ } catch (Exception e) { log.error("Failed download", e); throw e; @@ -73,7 +68,7 @@ public void run() { } catch (Exception e) { log.error("Failed download", e); file.updateProgress(0); - file.setSuccessIcon(MD5TableCell.ERROR_ICON); + file.setSuccessIcon(MD5TableCell.ERROR_ICON + ":" + e.getMessage()); try { new File(file.getLocalPath()).delete(); } catch (Exception ex) { @@ -82,7 +77,7 @@ public void run() { this.failed(); throw e; } finally { - couuntdownLatch(); + countdownLatch(); } } @@ -90,25 +85,15 @@ public void run() { protected void cancelled() { super.cancelled(); try { - double progress = this.getProgress(); -// if (!file.isDownloaded()) { downloadService.abortDownload(); -// File downloadFile = new File(getSaveLocation() + File.separator + getName()); -// boolean deleted = downloadFile.delete(); -// if (!deleted) { -// downloadFile.deleteOnExit(); -// } -// } else if (progress < 0) { -// this.updateProgress(0); -// } } catch (Exception e) { log.error("Failed cancel:", e); } finally { - couuntdownLatch(); + countdownLatch(); } } - private void couuntdownLatch() { + private void countdownLatch() { if (!countedDown) { countedDown = true; try { diff --git a/src/main/java/uk/ac/ebi/ena/downloader/gui/ResultsController.java b/src/main/java/uk/ac/ebi/ena/downloader/gui/ResultsController.java index e3975b8..1389db8 100644 --- a/src/main/java/uk/ac/ebi/ena/downloader/gui/ResultsController.java +++ b/src/main/java/uk/ac/ebi/ena/downloader/gui/ResultsController.java @@ -528,7 +528,6 @@ public void run() { } new Thread(() -> { try { - log.info("latch waiting"); latch.await(); int count = 0; for (int r = 0; r < notDoneFiles.size(); r++) { diff --git a/src/main/java/uk/ac/ebi/ena/downloader/gui/custom/MD5TableCell.java b/src/main/java/uk/ac/ebi/ena/downloader/gui/custom/MD5TableCell.java index 21fc9c2..757220d 100644 --- a/src/main/java/uk/ac/ebi/ena/downloader/gui/custom/MD5TableCell.java +++ b/src/main/java/uk/ac/ebi/ena/downloader/gui/custom/MD5TableCell.java @@ -55,9 +55,12 @@ protected void updateItem(String item, boolean empty) { setText(null); setGraphic(null); } else { - if (ERROR_ICON.equals(item)) { - imageView.setImage(new Image(item + ".png")); - Tooltip.install(imageView, new Tooltip("File did not download correctly. Please try again later.")); + if (item.startsWith(ERROR_ICON)) { + String[] parts = item.split("\\:"); + imageView.setImage(new Image(parts[0] + ".png")); + Tooltip tooltip = new Tooltip(parts.length > 1 ? StringUtils.substringAfter(item, ":").trim() : "File did not download correctly. Please try again later."); + tooltip.setWrapText(true); + Tooltip.install(imageView, tooltip); } else if (SUCCESS_ICON.equals(item)) { imageView.setImage(new Image(item + ".png")); Tooltip.install(imageView, new Tooltip("File fully downloaded. MD5 checksum verified.")); diff --git a/src/main/java/uk/ac/ebi/ena/downloader/service/DownloadService.java b/src/main/java/uk/ac/ebi/ena/downloader/service/DownloadService.java index 0c5b119..6d8e0e0 100644 --- a/src/main/java/uk/ac/ebi/ena/downloader/service/DownloadService.java +++ b/src/main/java/uk/ac/ebi/ena/downloader/service/DownloadService.java @@ -13,6 +13,7 @@ import java.io.File; import java.io.FileInputStream; +import java.net.URLEncoder; /** * Created by suranj on 31/05/2016. @@ -41,7 +42,7 @@ public Void downloadFileAspera(final RemoteFile remoteFile, DownloadSettings dow } public boolean fileAlreadyDownloaded(RemoteFile remoteFile) throws Exception { - final File downloadFile = new File(remoteFile.getSaveLocation() + File.separator + remoteFile.getName()); + final File downloadFile = new File(remoteFile.getSaveLocation() + File.separator + URLEncoder.encode(remoteFile.getName(), "UTF-8")); if (downloadFile.exists()) { // try { FileInputStream fis = new FileInputStream(downloadFile); diff --git a/src/main/java/uk/ac/ebi/ena/downloader/service/WarehouseQuery.java b/src/main/java/uk/ac/ebi/ena/downloader/service/WarehouseQuery.java index 5c53669..517a509 100644 --- a/src/main/java/uk/ac/ebi/ena/downloader/service/WarehouseQuery.java +++ b/src/main/java/uk/ac/ebi/ena/downloader/service/WarehouseQuery.java @@ -24,7 +24,11 @@ public class WarehouseQuery { public Map> doWarehouseSearch(String acc, DownloadSettings.Method method) { WarehouseQuery warehouseQuery = new WarehouseQuery(); - Map> map = warehouseQuery.query(acc, method); + String resultDomain = getResultDomain(acc); + Map> map = warehouseQuery.query(acc, method, resultDomain); + if (map.isEmpty()) { + map = warehouseQuery.query(acc, method, resultDomain.equals("analysis") ? "read_run" : "analysis"); + } return map; } @@ -34,23 +38,16 @@ public Map> doPortalSearch(String result, String query, return map; } - public Map> query(String accession, DownloadSettings.Method method) { + public Map> query(String accession, DownloadSettings.Method method, String resultDomain) { // URL stump for programmatic query of files - String resultDomain = getResultDomain(accession); String[] types = {}; String fields = ""; - String accField = ""; if (resultDomain.equals("analysis")) { types = new String[]{"submitted"}; - accField = "analysis_accession"; } else { types = new String[]{"fastq", "submitted", "sra"}; - accField = "run_accession"; } - fields += accField + ","; - - for (int t = 0; t < types.length; t++) { String type = types[t]; fields += type + "_" + method.name().toLowerCase() + "," + type + "_bytes," + type + "_md5"; @@ -58,7 +55,7 @@ public Map> query(String accession, DownloadSettings.Me fields += ","; } } - String url = "http://www.ebi.ac.uk/ena/data/warehouse/filereport?accession=" + accession + "&result=" + resultDomain + "&fields=" + fields; + String url = "https://www.ebi.ac.uk/ena/portal/api/filereport?accession=" + accession + "&result=" + resultDomain + "&fields=" + fields; log.info(url); try { // Build URL, Connect and get results reader diff --git a/src/main/java/uk/ac/ebi/ena/downloader/service/aspera/AsperaUtility.java b/src/main/java/uk/ac/ebi/ena/downloader/service/aspera/AsperaUtility.java index 48f9cd3..a3df3b0 100644 --- a/src/main/java/uk/ac/ebi/ena/downloader/service/aspera/AsperaUtility.java +++ b/src/main/java/uk/ac/ebi/ena/downloader/service/aspera/AsperaUtility.java @@ -2,8 +2,8 @@ import it.sauronsoftware.ftp4j.FTPClient; +import javafx.scene.control.ProgressIndicator; import org.apache.commons.codec.digest.DigestUtils; -import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -49,6 +49,8 @@ public AsperaUtility() { public void downloadFile(final RemoteFile remoteFile, DownloadSettings downloadSettings) throws Exception { try { + // aspera doesn't support url encoded file paths + // e.g. fasp.sra.ebi.ac.uk/vol1/ERA413/ERA413485/tab/TARA_018_DCM_%3C-0.22.gene.tsv for fasp.sra.ebi.ac.uk/vol1/ERA413/ERA413485/tab/TARA_018_DCM_<-0.22.gene.tsv final File downloadFile = new File(remoteFile.getSaveLocation() + File.separator + remoteFile.getName()); remoteFile.setLocalPath(downloadFile.getAbsolutePath()); @@ -74,6 +76,10 @@ public void downloadFile(final RemoteFile remoteFile, DownloadSettings downloadS String s = null; while ((s = reader.readLine()) != null) { log.info(s); + if (s.contains("Error")) { + remoteFile.updateProgress(ProgressIndicator.INDETERMINATE_PROGRESS); + throw new AsperaException(s); + } if (StringUtils.contains(s, "Store key in cache? (y/n)")) { process.getOutputStream().write("y".getBytes()); } @@ -94,7 +100,7 @@ public void downloadFile(final RemoteFile remoteFile, DownloadSettings downloadS if (!StringUtils.equals(md5, remoteFile.getMd5())) { log.debug("MD5 Error"); remoteFile.updateProgress(0); - remoteFile.setSuccessIcon(MD5TableCell.ERROR_ICON); + remoteFile.setSuccessIcon(MD5TableCell.ERROR_ICON + ":" + "MD5 Checksum verification failed."); try { new File(remoteFile.getLocalPath()).delete(); } catch (Exception e) { @@ -124,13 +130,6 @@ public void downloadFile(final RemoteFile remoteFile, DownloadSettings downloadS } reader.close(); - BufferedReader stdError = new BufferedReader(new InputStreamReader(process.getErrorStream())); - // Read command errors - String err = IOUtils.toString(stdError); - if (StringUtils.isNotBlank(err)) { - throw new AsperaException(err); - } - } catch (IOException ex) { log.error("IO error", ex); throw new AsperaException("Error downloading file: " + ex.getMessage()); diff --git a/src/main/java/uk/ac/ebi/ena/downloader/service/ftp/CommonsFTPUtility.java b/src/main/java/uk/ac/ebi/ena/downloader/service/ftp/CommonsFTPUtility.java index b6a194a..3e4a350 100644 --- a/src/main/java/uk/ac/ebi/ena/downloader/service/ftp/CommonsFTPUtility.java +++ b/src/main/java/uk/ac/ebi/ena/downloader/service/ftp/CommonsFTPUtility.java @@ -14,6 +14,7 @@ import java.io.*; import java.net.SocketTimeoutException; +import java.net.URLEncoder; /** * A utility class that provides functionality for downloading files from a FTP @@ -32,14 +33,6 @@ public class CommonsFTPUtility { private String password = "1234"; private FTPClient ftpClient = null; - private int replyCode; - - private InputStream inputStream; - - /*public FTP4JUtility() { - FTPClientConfig conf = new FTPClientConfig(FTPClientConfig.SYST_L8); - ftpClient.configure(conf); - }*/ /** * Connect and login to the server. @@ -83,7 +76,7 @@ public void connect() throws Exception { */ public synchronized void downloadFile(final RemoteFile remoteFile) throws Exception { try { - final File downloadFile = new File(remoteFile.getSaveLocation() + File.separator + remoteFile.getName()); + final File downloadFile = new File(remoteFile.getSaveLocation() + File.separator + URLEncoder.encode(remoteFile.getName(), "UTF-8")); log.debug(downloadFile.getAbsolutePath() + ":downloadFile.canWrite():" + downloadFile.canWrite()); log.debug("remote file:" + remoteFile.getPath()); // ftpClient.setType(FTPClient.TYPE_AUTO); @@ -146,20 +139,6 @@ public void run() { } }.start(); - /* - InputStream inputStream = null; - while (inputStream == null) { - try { - inputStream = ftpClient.retrieveFileStream(path); - log.debug("input stream:" + ftpClient.getReplyString()); - } catch (Exception e) { - e.log.error(); - log.debug("waiting 5 sec"); - wait(5000); - } - } - log.debug("input stream:" + ftpClient.getReplyString());*/ -// Util.copyStream(inputStream, outputStream2, (int) remoteFile.getTransferred(), remoteFile.getSize(), copyStreamListener); } catch (SocketTimeoutException ste) { throw ste; @@ -211,7 +190,6 @@ public void run() { } } catch (Exception e) { log.error("Abort fault:" + e.getMessage()); -// throw e; } } }.start(); diff --git a/src/main/java/uk/ac/ebi/ena/downloader/service/ftp/FTP4JUtility.java b/src/main/java/uk/ac/ebi/ena/downloader/service/ftp/FTP4JUtility.java index c652e1c..c6ebc99 100644 --- a/src/main/java/uk/ac/ebi/ena/downloader/service/ftp/FTP4JUtility.java +++ b/src/main/java/uk/ac/ebi/ena/downloader/service/ftp/FTP4JUtility.java @@ -15,6 +15,7 @@ import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; +import java.net.URLEncoder; /** * A utility class that provides functionality for downloading files from a FTP @@ -68,13 +69,13 @@ public void connect() throws Exception { public void downloadFile(final RemoteFile remoteFile) throws Exception { try { - final File downloadFile = new File(remoteFile.getSaveLocation() + File.separator + remoteFile.getName()); + final File downloadFile = new File(remoteFile.getSaveLocation() + File.separator + URLEncoder.encode(remoteFile.getName(), "UTF-8")); log.debug(downloadFile.getAbsolutePath() + ":downloadFile.canWrite():" + downloadFile.canWrite()); ftpClient.setType(FTPClient.TYPE_AUTO); String path = StringUtils.substringAfter(remoteFile.getPath(), this.host); String dir = StringUtils.substringAfter(StringUtils.substringBeforeLast(path, "/"), "/"); - ftpClient.changeDirectory(dir); +// ftpClient.changeDirectory(dir); // long fileSize = getFileSize(dir, StringUtils.substringAfterLast(path, "/")); // log.debug(fileSize); log.debug("path:" + path); @@ -107,7 +108,7 @@ public void run() { if (!StringUtils.equals(md5, remoteFile.getMd5())) { log.debug("MD5 Error"); remoteFile.updateProgress(0); - remoteFile.setSuccessIcon(MD5TableCell.ERROR_ICON); + remoteFile.setSuccessIcon(MD5TableCell.ERROR_ICON + ":" + "MD5 Checksum verification failed."); try { new File(remoteFile.getLocalPath()).delete(); } catch (Exception e) {