v1.1.1. Handling for illegal chars in file name. portal api for acc s…

…earch. fallback for other result type.
enasequence · Aug 25, 2017 · 4db0ee7 · 4db0ee7
1 parent 6fb851a
commit 4db0ee7
Show file tree

Hide file tree

Showing 9 changed files with 37 additions and 74 deletions.
diff --git a/README.md b/README.md
@@ -1,4 +1,4 @@
-# ENA File Downloader version 1.1
+# ENA File Downloader version 1.1.1
 
 Graphical user interface (GUI) for bulk downloading of run/analysis files from ENA via FTP or Aspera.
 
@@ -9,9 +9,9 @@ Copyright 2017 EMBL - European Bioinformatics Institute Licensed under the Apach
 You may obtain a copy of the License at: http://www.apache.org/licenses/LICENSE-2.0
 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
 
-# Installation
+# Download
 
-Download the latest release (.zip file) from the Releases section and extract it to a location of your choice. The archive contains an executable jar which contains all dependencies.
+Download the latest release (.zip file) from the Releases section of [the GitHub project](https://github.com/enasequence/ena-ftp-downloader) and extract it to a location of your choice. The archive contains an executable jar which contains all dependencies.
 
 # Dependencies
 

diff --git a/src/main/java/uk/ac/ebi/ena/downloader/gui/DownloadTask.java b/src/main/java/uk/ac/ebi/ena/downloader/gui/DownloadTask.java
@@ -15,7 +15,7 @@
  * Created by suranj on 01/08/2016.
  */
 public class DownloadTask extends Task<Void> {
-    private final static int RETRY_COUNT = 10;
+    private final static int RETRY_COUNT = 5;
     private final static Logger log = LoggerFactory.getLogger(DownloadTask.class);
 
     private final RemoteFile file;
@@ -58,12 +58,7 @@ protected Void call() throws Exception {
                 log.debug("calling success:" + file.getName());
                 file.setDownloaded(true);
                 succeeded();
-                /*Platform.runLater(new Runnable() {
-                    @Override
-                    public void run() {
 
-                    }
-                });*/
             } catch (Exception e) {
                 log.error("Failed download", e);
                 throw e;
@@ -73,7 +68,7 @@ public void run() {
         } catch (Exception e) {
             log.error("Failed download", e);
             file.updateProgress(0);
-            file.setSuccessIcon(MD5TableCell.ERROR_ICON);
+            file.setSuccessIcon(MD5TableCell.ERROR_ICON + ":" + e.getMessage());
             try {
                 new File(file.getLocalPath()).delete();
             } catch (Exception ex) {
@@ -82,33 +77,23 @@ public void run() {
             this.failed();
             throw e;
         } finally {
-            couuntdownLatch();
+            countdownLatch();
         }
     }
 
     @Override
     protected void cancelled() {
         super.cancelled();
         try {
-            double progress = this.getProgress();
-//            if (!file.isDownloaded()) {
             downloadService.abortDownload();
-//                File downloadFile = new File(getSaveLocation() + File.separator + getName());
-//                boolean deleted = downloadFile.delete();
-//                if (!deleted) {
-//                    downloadFile.deleteOnExit();
-//                }
-//            } else if (progress < 0) {
-//                this.updateProgress(0);
-//            }
         } catch (Exception e) {
             log.error("Failed cancel:", e);
         } finally {
-            couuntdownLatch();
+            countdownLatch();
         }
     }
 
-    private void couuntdownLatch() {
+    private void countdownLatch() {
         if (!countedDown) {
             countedDown = true;
             try {

diff --git a/src/main/java/uk/ac/ebi/ena/downloader/gui/ResultsController.java b/src/main/java/uk/ac/ebi/ena/downloader/gui/ResultsController.java
@@ -528,7 +528,6 @@ public void run() {
             }
             new Thread(() -> {
                 try {
-                    log.info("latch waiting");
                     latch.await();
                     int count = 0;
                     for (int r = 0; r < notDoneFiles.size(); r++) {

diff --git a/src/main/java/uk/ac/ebi/ena/downloader/gui/custom/MD5TableCell.java b/src/main/java/uk/ac/ebi/ena/downloader/gui/custom/MD5TableCell.java
@@ -55,9 +55,12 @@ protected void updateItem(String item, boolean empty) {
             setText(null);
             setGraphic(null);
         } else {
-            if (ERROR_ICON.equals(item)) {
-                imageView.setImage(new Image(item + ".png"));
-                Tooltip.install(imageView, new Tooltip("File did not download correctly. Please try again later."));
+            if (item.startsWith(ERROR_ICON)) {
+                String[] parts = item.split("\\:");
+                imageView.setImage(new Image(parts[0] + ".png"));
+                Tooltip tooltip = new Tooltip(parts.length > 1 ? StringUtils.substringAfter(item, ":").trim() : "File did not download correctly. Please try again later.");
+                tooltip.setWrapText(true);
+                Tooltip.install(imageView, tooltip);
             } else if (SUCCESS_ICON.equals(item)) {
                 imageView.setImage(new Image(item + ".png"));
                 Tooltip.install(imageView, new Tooltip("File fully downloaded. MD5 checksum verified."));

diff --git a/src/main/java/uk/ac/ebi/ena/downloader/service/DownloadService.java b/src/main/java/uk/ac/ebi/ena/downloader/service/DownloadService.java
@@ -13,6 +13,7 @@
 
 import java.io.File;
 import java.io.FileInputStream;
+import java.net.URLEncoder;
 
 /**
  * Created by suranj on 31/05/2016.
@@ -41,7 +42,7 @@ public Void downloadFileAspera(final RemoteFile remoteFile, DownloadSettings dow
     }
 
     public boolean fileAlreadyDownloaded(RemoteFile remoteFile) throws Exception {
-        final File downloadFile = new File(remoteFile.getSaveLocation() + File.separator + remoteFile.getName());
+        final File downloadFile = new File(remoteFile.getSaveLocation() + File.separator + URLEncoder.encode(remoteFile.getName(), "UTF-8"));
         if (downloadFile.exists()) {
 //            try {
             FileInputStream fis = new FileInputStream(downloadFile);

diff --git a/src/main/java/uk/ac/ebi/ena/downloader/service/WarehouseQuery.java b/src/main/java/uk/ac/ebi/ena/downloader/service/WarehouseQuery.java
@@ -24,7 +24,11 @@ public class WarehouseQuery {
 
     public Map<String, List<RemoteFile>> doWarehouseSearch(String acc, DownloadSettings.Method method) {
         WarehouseQuery warehouseQuery = new WarehouseQuery();
-        Map<String, List<RemoteFile>> map = warehouseQuery.query(acc, method);
+        String resultDomain = getResultDomain(acc);
+        Map<String, List<RemoteFile>> map = warehouseQuery.query(acc, method, resultDomain);
+        if (map.isEmpty()) {
+            map = warehouseQuery.query(acc, method, resultDomain.equals("analysis") ? "read_run" : "analysis");
+        }
         return map;
     }
 
@@ -34,31 +38,24 @@ public Map<String, List<RemoteFile>> doPortalSearch(String result, String query,
         return map;
     }
 
-    public Map<String, List<RemoteFile>> query(String accession, DownloadSettings.Method method) {
+    public Map<String, List<RemoteFile>> query(String accession, DownloadSettings.Method method, String resultDomain) {
         // URL stump for programmatic query of files
-        String resultDomain = getResultDomain(accession);
         String[] types = {};
         String fields = "";
-        String accField = "";
         if (resultDomain.equals("analysis")) {
             types = new String[]{"submitted"};
-            accField = "analysis_accession";
         } else {
             types = new String[]{"fastq", "submitted", "sra"};
-            accField = "run_accession";
         }
 
-        fields += accField + ",";
-
-
         for (int t = 0; t < types.length; t++) {
             String type = types[t];
             fields += type + "_" + method.name().toLowerCase() + "," + type + "_bytes," + type + "_md5";
             if (t < types.length - 1) {
                 fields += ",";
             }
         }
-        String url = "http://www.ebi.ac.uk/ena/data/warehouse/filereport?accession=" + accession + "&result=" + resultDomain + "&fields=" + fields;
+        String url = "https://www.ebi.ac.uk/ena/portal/api/filereport?accession=" + accession + "&result=" + resultDomain + "&fields=" + fields;
         log.info(url);
         try {
             // Build URL, Connect and get results reader

diff --git a/src/main/java/uk/ac/ebi/ena/downloader/service/aspera/AsperaUtility.java b/src/main/java/uk/ac/ebi/ena/downloader/service/aspera/AsperaUtility.java
@@ -2,8 +2,8 @@
 
 
 import it.sauronsoftware.ftp4j.FTPClient;
+import javafx.scene.control.ProgressIndicator;
 import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -49,6 +49,8 @@ public AsperaUtility() {
     public void downloadFile(final RemoteFile remoteFile, DownloadSettings downloadSettings) throws Exception {
         try {
 
+            // aspera doesn't support url encoded file paths
+            // e.g. fasp.sra.ebi.ac.uk/vol1/ERA413/ERA413485/tab/TARA_018_DCM_%3C-0.22.gene.tsv for fasp.sra.ebi.ac.uk/vol1/ERA413/ERA413485/tab/TARA_018_DCM_<-0.22.gene.tsv
             final File downloadFile = new File(remoteFile.getSaveLocation() + File.separator + remoteFile.getName());
 
             remoteFile.setLocalPath(downloadFile.getAbsolutePath());
@@ -74,6 +76,10 @@ public void downloadFile(final RemoteFile remoteFile, DownloadSettings downloadS
             String s = null;
             while ((s = reader.readLine()) != null) {
                 log.info(s);
+                if (s.contains("Error")) {
+                    remoteFile.updateProgress(ProgressIndicator.INDETERMINATE_PROGRESS);
+                    throw new AsperaException(s);
+                }
                 if (StringUtils.contains(s, "Store key in cache? (y/n)")) {
                     process.getOutputStream().write("y".getBytes());
                 }
@@ -94,7 +100,7 @@ public void downloadFile(final RemoteFile remoteFile, DownloadSettings downloadS
                                     if (!StringUtils.equals(md5, remoteFile.getMd5())) {
                                         log.debug("MD5 Error");
                                         remoteFile.updateProgress(0);
-                                        remoteFile.setSuccessIcon(MD5TableCell.ERROR_ICON);
+                                        remoteFile.setSuccessIcon(MD5TableCell.ERROR_ICON + ":" + "MD5 Checksum verification failed.");
                                         try {
                                             new File(remoteFile.getLocalPath()).delete();
                                         } catch (Exception e) {
@@ -124,13 +130,6 @@ public void downloadFile(final RemoteFile remoteFile, DownloadSettings downloadS
             }
             reader.close();
 
-            BufferedReader stdError = new BufferedReader(new InputStreamReader(process.getErrorStream()));
-            // Read command errors
-            String err = IOUtils.toString(stdError);
-            if (StringUtils.isNotBlank(err)) {
-                throw new AsperaException(err);
-            }
-
         } catch (IOException ex) {
             log.error("IO error", ex);
             throw new AsperaException("Error downloading file: " + ex.getMessage());

diff --git a/src/main/java/uk/ac/ebi/ena/downloader/service/ftp/CommonsFTPUtility.java b/src/main/java/uk/ac/ebi/ena/downloader/service/ftp/CommonsFTPUtility.java
@@ -14,6 +14,7 @@
 
 import java.io.*;
 import java.net.SocketTimeoutException;
+import java.net.URLEncoder;
 
 /**
  * A utility class that provides functionality for downloading files from a FTP
@@ -32,14 +33,6 @@ public class CommonsFTPUtility {
     private String password = "1234";
 
     private FTPClient ftpClient = null;
-    private int replyCode;
-
-    private InputStream inputStream;
-
-    /*public FTP4JUtility() {
-        FTPClientConfig conf = new FTPClientConfig(FTPClientConfig.SYST_L8);
-        ftpClient.configure(conf);
-    }*/
 
     /**
      * Connect and login to the server.
@@ -83,7 +76,7 @@ public void connect() throws Exception {
      */
     public synchronized void downloadFile(final RemoteFile remoteFile) throws Exception {
         try {
-            final File downloadFile = new File(remoteFile.getSaveLocation() + File.separator + remoteFile.getName());
+            final File downloadFile = new File(remoteFile.getSaveLocation() + File.separator + URLEncoder.encode(remoteFile.getName(), "UTF-8"));
             log.debug(downloadFile.getAbsolutePath() + ":downloadFile.canWrite():" + downloadFile.canWrite());
             log.debug("remote file:" + remoteFile.getPath());
 //            ftpClient.setType(FTPClient.TYPE_AUTO);
@@ -146,20 +139,6 @@ public void run() {
                 }
             }.start();
 
-       /*
-            InputStream inputStream = null;
-            while (inputStream == null) {
-                try {
-                    inputStream = ftpClient.retrieveFileStream(path);
-                    log.debug("input stream:" + ftpClient.getReplyString());
-                } catch (Exception e) {
-                    e.log.error();
-                    log.debug("waiting 5 sec");
-                    wait(5000);
-                }
-            }
-            log.debug("input stream:" + ftpClient.getReplyString());*/
-//            Util.copyStream(inputStream, outputStream2, (int) remoteFile.getTransferred(), remoteFile.getSize(), copyStreamListener);
 
         } catch (SocketTimeoutException ste) {
             throw ste;
@@ -211,7 +190,6 @@ public void run() {
                     }
                 } catch (Exception e) {
                     log.error("Abort fault:" + e.getMessage());
-//            throw e;
                 }
             }
         }.start();

diff --git a/src/main/java/uk/ac/ebi/ena/downloader/service/ftp/FTP4JUtility.java b/src/main/java/uk/ac/ebi/ena/downloader/service/ftp/FTP4JUtility.java
@@ -15,6 +15,7 @@
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.net.URLEncoder;
 
 /**
  * A utility class that provides functionality for downloading files from a FTP
@@ -68,13 +69,13 @@ public void connect() throws Exception {
     public void downloadFile(final RemoteFile remoteFile) throws Exception {
         try {
 
-            final File downloadFile = new File(remoteFile.getSaveLocation() + File.separator + remoteFile.getName());
+            final File downloadFile = new File(remoteFile.getSaveLocation() + File.separator + URLEncoder.encode(remoteFile.getName(), "UTF-8"));
             log.debug(downloadFile.getAbsolutePath() + ":downloadFile.canWrite():" + downloadFile.canWrite());
 
             ftpClient.setType(FTPClient.TYPE_AUTO);
             String path = StringUtils.substringAfter(remoteFile.getPath(), this.host);
             String dir = StringUtils.substringAfter(StringUtils.substringBeforeLast(path, "/"), "/");
-            ftpClient.changeDirectory(dir);
+//            ftpClient.changeDirectory(dir);
 //            long fileSize = getFileSize(dir, StringUtils.substringAfterLast(path, "/"));
 //            log.debug(fileSize);
             log.debug("path:" + path);
@@ -107,7 +108,7 @@ public void run() {
                                     if (!StringUtils.equals(md5, remoteFile.getMd5())) {
                                         log.debug("MD5 Error");
                                         remoteFile.updateProgress(0);
-                                        remoteFile.setSuccessIcon(MD5TableCell.ERROR_ICON);
+                                        remoteFile.setSuccessIcon(MD5TableCell.ERROR_ICON  + ":" + "MD5 Checksum verification failed.");
                                         try {
                                             new File(remoteFile.getLocalPath()).delete();
                                         } catch (Exception e) {