diff --git a/lib/square/okhttp-3.10.0.jar b/lib/square/okhttp-3.10.0.jar new file mode 100644 index 0000000..1c3cfb8 Binary files /dev/null and b/lib/square/okhttp-3.10.0.jar differ diff --git a/lib/square/okio-1.14.0.jar b/lib/square/okio-1.14.0.jar new file mode 100644 index 0000000..a0ee347 Binary files /dev/null and b/lib/square/okio-1.14.0.jar differ diff --git a/src/org/helioviewer/jhv/base/FileUtils.java b/src/org/helioviewer/jhv/base/FileUtils.java deleted file mode 100644 index 6ab64db..0000000 --- a/src/org/helioviewer/jhv/base/FileUtils.java +++ /dev/null @@ -1,130 +0,0 @@ -package org.helioviewer.jhv.base; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.OutputStream; -import java.net.URL; -import java.nio.charset.StandardCharsets; -import java.util.Scanner; - -// A class which provides functions for accessing and working with files -public class FileUtils { - - private static final int BUFSIZ = 65536; - - /** - * Return the current working directory - * - * @return the current working directory - */ - public static File getWorkingDirectory() { - return new File(System.getProperty("user.dir")); - } - - /** - * Method copies a file from src to dst. - * - * @param src - * Source file - * @param dst - * Destination file - * @throws IOException - */ - public static void copy(File src, File dst) throws IOException { - try (InputStream in = new FileInputStream(src); OutputStream out = new FileOutputStream(dst)) { - // Transfer bytes from in to out - byte[] buf = new byte[BUFSIZ]; - int len; - while ((len = in.read(buf)) > 0) { - out.write(buf, 0, len); - } - } - } - - /** - * Method saving a stream to dst. - * - * @param in - * Input stream, will be closed if finished - * @param dst - * Destination file - * @throws IOException - */ - public static void save(InputStream in, File dst) throws IOException { - // Transfer bytes from in to out - try (OutputStream out = new FileOutputStream(dst)) { - byte[] buf = new byte[BUFSIZ]; - int len; - while ((len = in.read(buf)) > 0) { - out.write(buf, 0, len); - } - } - in.close(); - } - - /** - * Returns an input stream to a resource. This function can be used even if - * the whole program and resources are within a JAR file. The path must - * begin with a slash and contain all subfolders, e.g.: /images/sample_image.png - * The class loader used is the same which was used to load FileUtils. - * - * @param resourcePath - * The path to the resource - * @return An InputStream to the resource - */ - public static InputStream getResourceInputStream(String resourcePath) { - return FileUtils.class.getResourceAsStream(resourcePath); - } - - public static String convertStreamToString(InputStream is) { - try (Scanner s = new Scanner(is, StandardCharsets.UTF_8.name())) { - s.useDelimiter("\\A"); - return s.hasNext() ? s.next() : ""; - } - } - - /** - * Returns an URL to a resource. This function can be used even if the whole - * program and resources are within a JAR file. The path must begin with a - * slash and contain all subfolders, e.g.: /images/sample_image.png - * The class loader used is the same which was used to load FileUtils. - * - * @param resourcePath - * The path to the resource - * @return An URL to the resource - */ - public static URL getResourceUrl(String resourcePath) { - return FileUtils.class.getResource(resourcePath); - } - - public static boolean deleteDir(File dir) { - String[] children; - if (dir.isDirectory() && (children = dir.list()) != null) { - for (String child : children) { - boolean success = deleteDir(new File(dir, child)); - if (!success) { - return false; - } - } - } - return dir.delete(); - } - - public static String URL2String(URL url) { - StringBuilder sb = new StringBuilder(); - try (Scanner scanner = new Scanner(new BufferedReader(new InputStreamReader(url.openStream(), StandardCharsets.UTF_8)))) { - while (scanner.hasNext()) { - sb.append(scanner.nextLine()).append('\n'); - } - } catch (Exception e) { - e.printStackTrace(); - } - return sb.toString(); - } - -} diff --git a/src/org/helioviewer/jhv/base/JSONUtils.java b/src/org/helioviewer/jhv/base/JSONUtils.java deleted file mode 100644 index 2f2a684..0000000 --- a/src/org/helioviewer/jhv/base/JSONUtils.java +++ /dev/null @@ -1,38 +0,0 @@ -package org.helioviewer.jhv.base; - -import java.io.ByteArrayOutputStream; -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.OutputStreamWriter; -import java.nio.charset.StandardCharsets; -import java.util.zip.GZIPOutputStream; - -import org.helioviewer.jhv.log.Log; -import org.json.JSONObject; -import org.json.JSONTokener; - -public class JSONUtils { - - private static final int BUFSIZ = 65536; - - public static JSONObject getJSONStream(InputStream in) { - try (BufferedReader reader = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8), BUFSIZ)) { - return new JSONObject(new JSONTokener(reader)); - } catch (Exception e) { - Log.error("Invalid JSON response " + e); - return new JSONObject(); - } - } - - public static byte[] compressJSON(JSONObject json) throws IOException { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - OutputStreamWriter out = new OutputStreamWriter(new GZIPOutputStream(baos, BUFSIZ), StandardCharsets.UTF_8); - json.write(out); - out.close(); - - return baos.toByteArray(); - } - -} diff --git a/src/org/helioviewer/jhv/base/Regex.java b/src/org/helioviewer/jhv/base/Regex.java index 6ed9cd7..1ec80a5 100644 --- a/src/org/helioviewer/jhv/base/Regex.java +++ b/src/org/helioviewer/jhv/base/Regex.java @@ -70,7 +70,9 @@ public class Regex { public static final Pattern FloatingPoint = Pattern.compile("[\\x00-\\x20]*[+-]?(NaN|Infinity|((((\\p{Digit}+)(\\.)?((\\p{Digit}+)?)([eE][+-]?(\\p{Digit}+))?)|(\\.((\\p{Digit}+))([eE][+-]?(\\p{Digit}+))?)|(((0[xX](\\p{XDigit}+)(\\.)?)|(0[xX](\\p{XDigit}+)?(\\.)(\\p{XDigit}+)))[pP][+-]?(\\p{Digit}+)))[fFdD]?))[\\x00-\\x20]*"); public static final Pattern Integer = Pattern.compile("\\d+"); - // Pattern to extract the filename from HTTP Content-Disposition header - public static final Pattern ContentDispositionFilename = Pattern.compile("filename=\\\"(.*?)\\\""); + public static final Pattern Comma = Pattern.compile(","); + public static final Pattern Return = Pattern.compile("\n"); + public static final Pattern Space = Pattern.compile(" "); + public static final Pattern HttpField = Pattern.compile(": "); } diff --git a/src/org/helioviewer/jhv/io/DataSources.java b/src/org/helioviewer/jhv/io/DataSources.java index a134598..764e4a0 100644 --- a/src/org/helioviewer/jhv/io/DataSources.java +++ b/src/org/helioviewer/jhv/io/DataSources.java @@ -4,9 +4,12 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.Map; import java.util.Set; +import org.everit.json.schema.Validator; + @SuppressWarnings("serial") public class DataSources { @@ -14,9 +17,9 @@ public class DataSources { "SOHO", "SDO", "STEREO_A", "STEREO_B", "PROBA2", "ROB-USET", "ROB-Humain", "NSO-GONG", "NSO-SOLIS", "Kanzelhoehe", "NRH", "Yohkoh", "Hinode", "TRACE" ))); - private static final HashMap> serverSettings = new HashMap>() { + private static final Map> serverSettings = Collections.unmodifiableMap(new LinkedHashMap>() { { - put("ROB", new HashMap() { + put("ROB", Collections.unmodifiableMap(new HashMap() { { put("API.getDataSources", "http://swhv.oma.be/hv/api/?action=getDataSources&verbose=true&enable=[STEREO_A,STEREO_B,PROBA2]"); put("API.getJP2Image", "http://swhv.oma.be/hv/api/index.php?action=getJP2Image&"); @@ -25,8 +28,18 @@ public class DataSources { put("schema", "/data/sources_v1.0.json"); put("availability.images", "http://swhv.oma.be/availability/images/availability/availability.html"); } - }); - put("IAS", new HashMap() { + })); +/* put("ROB Test", Collections.unmodifiableMap(new HashMap() { + { + put("API.getDataSources", "http://swhv2.oma.be:8083/index.php?action=getDataSources&verbose=true&enable=[STEREO_A,STEREO_B,PROBA2]"); + put("API.getJP2Image", "http://swhv2.oma.be:8083/index.php?action=getJP2Image&"); + put("API.getJPX", "http://swhv2.oma.be:8083/index.php?action=getJPX&"); + put("label", "Royal Observatory of Belgium"); + put("schema", "/data/sources_v1.0.json"); + put("availability.images", "http://swhv2.oma.be/availability/images/availability/availability.html"); + } + })); */ + put("IAS", Collections.unmodifiableMap(new HashMap() { { put("API.getDataSources", "https://helioviewer-api.ias.u-psud.fr/v2/getDataSources/?verbose=true&enable=[TRACE,Hinode,Yohkoh,STEREO_A,STEREO_B,PROBA2]"); put("API.getJP2Image", "https://helioviewer-api.ias.u-psud.fr/v2/getJP2Image/?"); @@ -34,8 +47,17 @@ public class DataSources { put("label", "Institut d'Astrophysique Spatiale"); put("schema", "/data/sources_v1.0.json"); } - }); - put("GSFC", new HashMap() { + })); +/* put("IAS Test", Collections.unmodifiableMap(new HashMap() { + { + put("API.getDataSources", "https://inf-helio-test-api.ias.u-psud.fr/v2/getDataSources/?verbose=true&enable=[TRACE,Hinode,Yohkoh,STEREO_A,STEREO_B,PROBA2]"); + put("API.getJP2Image", "https://inf-helio-test-api.ias.u-psud.fr/v2/getJP2Image/?"); + put("API.getJPX", "https://inf-helio-test-api.ias.u-psud.fr/v2/getJPX/?"); + put("label", "Institut d'Astrophysique Spatiale"); + put("schema", "/data/sources_v1.0.json"); + } + })); */ + put("GSFC", Collections.unmodifiableMap(new HashMap() { { put("API.getDataSources", "https://api.helioviewer.org/v2/getDataSources/?verbose=true&enable=[TRACE,Hinode,Yohkoh,STEREO_A,STEREO_B,PROBA2]"); put("API.getJP2Image", "https://api.helioviewer.org/v2/getJP2Image/?"); @@ -43,9 +65,8 @@ public class DataSources { put("label", "Goddard Space Flight Center"); put("schema", "/data/sources_v1.0.json"); } - }); - /* - put("GSFC SCI Test", new HashMap() { + })); +/* put("GSFC SCI Test", Collections.unmodifiableMap(new HashMap() { { put("API.getDataSources", "http://helioviewer.sci.gsfc.nasa.gov/api.php?action=getDataSources&verbose=true&enable=[TRACE,Hinode,Yohkoh,STEREO_A,STEREO_B,PROBA2]"); put("API.getJP2Image", "http://helioviewer.sci.gsfc.nasa.gov/api.php?action=getJP2Image&"); @@ -53,8 +74,8 @@ public class DataSources { put("label", "Goddard Space Flight Center SCI Test"); put("schema", "/data/sources_v1.0.json"); } - }); - put("GSFC NDC Test", new HashMap() { + })); + put("GSFC NDC Test", Collections.unmodifiableMap(new HashMap() { { put("API.getDataSources", "http://gs671-heliovw7.ndc.nasa.gov/api.php?action=getDataSources&verbose=true&enable=[TRACE,Hinode,Yohkoh,STEREO_A,STEREO_B,PROBA2]"); put("API.getJP2Image", "http://gs671-heliovw7.ndc.nasa.gov/api.php?action=getJP2Image&"); @@ -62,8 +83,8 @@ public class DataSources { put("label", "Goddard Space Flight Center NDC Test"); put("schema", "/data/sources_v1.0.json"); } - }); - put("LOCALHOST", new HashMap() { + })); */ +/* put("LOCALHOST", Collections.unmodifiableMap(new HashMap() { { put("API.getDataSources", "http://localhost:8080/helioviewer/api/?action=getDataSources&verbose=true&enable=[STEREO_A,STEREO_B,PROBA2]"); put("API.getJP2Image", "http://localhost:8080/helioviewer/api/index.php?action=getJP2Image&"); @@ -71,10 +92,9 @@ public class DataSources { put("schema", "/data/sources_v1.0.json"); put("label", "Localhost"); } - }); - */ + })); */ } - }; + }); public static Set getServers() { return serverSettings.keySet(); @@ -86,8 +106,9 @@ public static String getServerSetting(String server, String setting) { } public static void loadSources() { + Validator validator = Validator.builder().failEarly().build(); for (String serverName : serverSettings.keySet()) - new DataSourcesTask(serverName); + new DataSourcesTask(serverName, validator); } } diff --git a/src/org/helioviewer/jhv/io/DataSourcesTask.java b/src/org/helioviewer/jhv/io/DataSourcesTask.java index ce1c8b8..f303a74 100644 --- a/src/org/helioviewer/jhv/io/DataSourcesTask.java +++ b/src/org/helioviewer/jhv/io/DataSourcesTask.java @@ -2,22 +2,23 @@ import java.io.InputStream; +import org.helioviewer.jhv.log.Log; +import org.helioviewer.jhv.time.TimeUtils; + import org.everit.json.schema.Schema; +import org.everit.json.schema.Validator; import org.everit.json.schema.ValidationException; import org.everit.json.schema.loader.SchemaLoader; -import org.helioviewer.jhv.base.FileUtils; -import org.helioviewer.jhv.base.JSONUtils; -import org.helioviewer.jhv.log.Log; -import org.helioviewer.jhv.time.TimeUtils; import org.json.JSONObject; -import org.json.JSONTokener; public class DataSourcesTask implements Runnable { + private final Validator validator; private final String url; private final String schemaName; - public DataSourcesTask(String server) { + public DataSourcesTask(String server, Validator _validator) { + validator = _validator; url = DataSources.getServerSetting(server, "API.getDataSources"); schemaName = DataSources.getServerSetting(server, "schema"); Thread t = new Thread(this, server); @@ -26,12 +27,13 @@ public DataSourcesTask(String server) { @Override public void run() { - try (InputStream is = FileUtils.getResourceInputStream(schemaName)) { - JSONObject rawSchema = new JSONObject(new JSONTokener(is)); + try (InputStream is = FileUtils.getResource(schemaName)) { + JSONObject rawSchema = JSONUtils.get(is); SchemaLoader schemaLoader = SchemaLoader.builder().schemaJson(rawSchema).addFormatValidator(new TimeUtils.SQLDateTimeFormatValidator()).build(); Schema schema = schemaLoader.load().build(); - JSONObject json = JSONUtils.getJSONStream(new DownloadStream(url).getInput()); - schema.validate(json); + + JSONObject jo = JSONUtils.get(url); + validator.performValidation(schema, jo); } catch (ValidationException e) { Log.error("Server " + url + " " + e); e.getCausingExceptions().stream().map(ValidationException::getMessage).forEach(Log::error); diff --git a/src/org/helioviewer/jhv/io/DownloadStream.java b/src/org/helioviewer/jhv/io/DownloadStream.java deleted file mode 100644 index 94cc71c..0000000 --- a/src/org/helioviewer/jhv/io/DownloadStream.java +++ /dev/null @@ -1,207 +0,0 @@ -package org.helioviewer.jhv.io; - -import java.awt.EventQueue; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStreamWriter; -import java.net.HttpURLConnection; -import java.net.MalformedURLException; -import java.net.URL; -import java.net.URLConnection; -import java.nio.charset.StandardCharsets; -import java.util.regex.Matcher; -import java.util.zip.GZIPInputStream; -import java.util.zip.InflaterInputStream; - -import org.helioviewer.jhv.JHVGlobals; -import org.helioviewer.jhv.base.Regex; -import org.helioviewer.jhv.log.Log; - -public class DownloadStream { - - // Input stream to read the data from - private InputStream in = null; - - // Output to send as a post request - private String output = null; - - // Suggested name to save (if wanted) - private String outputName = null; - - private String contentDisposition = null; - private int contentLength = -1; - private boolean response400 = false; - - // Read timeout in ms - private final int readTimeout; - - // Connect timeout in ms - private final int connectTimeout; - - // URL to connect - private final URL url; - private final boolean ignore400; - - private DownloadStream(URL _url, int _connectTimeout, int _readTimeout, boolean _ignore400) { - url = _url; - readTimeout = _readTimeout; - connectTimeout = _connectTimeout; - ignore400 = _ignore400; - } - - private DownloadStream(URL _url, boolean _ignore400) { - this(_url, JHVGlobals.getStdConnectTimeout(), JHVGlobals.getStdReadTimeout(), _ignore400); - } - - public DownloadStream(String _url, boolean _ignore400) throws MalformedURLException { - this(new URL(_url), _ignore400); - } - - public DownloadStream(URL _url) { - this(_url, JHVGlobals.getStdConnectTimeout(), JHVGlobals.getStdReadTimeout(), false); - } - - public DownloadStream(String _url) throws MalformedURLException { - this(new URL(_url)); - } - - public boolean isResponse400() { - return response400; - } - - private static InputStream getEncodedStream(String encoding, InputStream httpStream) throws IOException { - if (encoding != null) { - switch (encoding.toLowerCase()) { - case "gzip": - return new GZIPInputStream(httpStream); - case "deflate": - return new InflaterInputStream(httpStream); - } - } - return httpStream; - } - - /** - * Opens the connection with compression if the server supports - * - * @throws IOException - * From accessing the network - */ - private void connect() throws IOException { - if (EventQueue.isDispatchThread()) - throw new IOException("Don't do that"); - - //Log.debug("Connect to " + url); - URLConnection connection = url.openConnection(); - // Set timeouts - connection.setConnectTimeout(connectTimeout); - connection.setReadTimeout(readTimeout); - - if (connection instanceof HttpURLConnection) { - HttpURLConnection httpC = (HttpURLConnection) connection; - // get compression if supported - httpC.setRequestProperty("Accept-Encoding", "gzip,deflate"); - httpC.setRequestProperty("User-Agent", JHVGlobals.userAgent); - - // Write post data if necessary - if (output != null) { - connection.setDoOutput(true); - try (OutputStreamWriter out = new OutputStreamWriter(connection.getOutputStream(), StandardCharsets.UTF_8)) { - out.write(output); - } - } - - try { - httpC.connect(); - } catch (IOException e) { - Log.warn("HTTP connection failed: " + url + " " + e); - } - - // Check the connection code - int code = httpC.getResponseCode(); - if (code > 400) { - throw new IOException("Error opening HTTP connection to " + url + " Response code: " + code); - } - - if (!ignore400 && code == 400) { - throw new IOException("Error opening HTTP connection to " + url + " Response code: " + code); - } - - InputStream strm; - if (code == 400) { - response400 = true; - strm = httpC.getErrorStream(); - if (strm == null) - strm = httpC.getInputStream(); - } else { - strm = httpC.getInputStream(); - } - - contentDisposition = httpC.getHeaderField("Content-Disposition"); - in = getEncodedStream(httpC.getContentEncoding(), strm); - } else { - // Not an http connection - // Write post data if necessary - if (output != null) { - connection.setDoOutput(true); - try (OutputStreamWriter out = new OutputStreamWriter(connection.getOutputStream(), StandardCharsets.UTF_8)) { - out.write(output); - } - } - // Okay just normal - in = connection.getInputStream(); - } - contentLength = connection.getContentLength(); - } - - /** - * Gives the outstream to read the response, after calling connect. If it is - * not already connected it will automatically connect - * - * @return output stream of the connection - * @throws IOException - * Error from creating the connction - */ - public InputStream getInput() throws IOException { - if (in == null) - connect(); - return in; - } - - /** - * After requesting the data the associated file name to save from - * Content-Disposition or the url name - * - * @return suggested download name - */ - public String getOutputName() { - if (outputName == null) { - if (contentDisposition != null) { - Matcher m = Regex.ContentDispositionFilename.matcher(contentDisposition); - if (m.find()) { - outputName = m.group(1); - } - } - if (outputName == null) { - outputName = url.getFile().replace('/', '-'); - } - } - return outputName; - } - - public int getContentLength() { - return contentLength; - } - - /** - * Set the output to send to the server (in HTTP as POST) - * - * @param _output - * Send output to the server, null if nothing (GET in HTTP) - */ - public void setOutput(String _output) { - output = _output; - } - -} diff --git a/src/org/helioviewer/jhv/io/FileUtils.java b/src/org/helioviewer/jhv/io/FileUtils.java new file mode 100644 index 0000000..66a7b86 --- /dev/null +++ b/src/org/helioviewer/jhv/io/FileUtils.java @@ -0,0 +1,82 @@ +package org.helioviewer.jhv.io; + +import java.io.File; +import java.io.FileFilter; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; + +import okio.Okio; +import okio.BufferedSource; + +public class FileUtils { + + public static InputStream getResource(String path) throws IOException { + InputStream is = FileUtils.class.getResourceAsStream(path); + if (is == null) + throw new IOException("Resource " + path + " not found"); + return is; + } + + public static String streamToString(InputStream is) throws IOException { + try (BufferedSource buffer = Okio.buffer(Okio.source(is))) { + return buffer.readString(StandardCharsets.UTF_8); + } + } + + private static final SimpleFileVisitor nukeVisitor = new SimpleFileVisitor() { + + private FileVisitResult delete(Path file) throws IOException { + Files.delete(file); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + return delete(file); + } + + @Override + public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException { + return delete(file); + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { + return delete(dir); + } + + }; + + public static void deleteDir(File dir) throws IOException { + Files.walkFileTree(dir.toPath(), nukeVisitor); + } + + public static File tempDir(File parent, String name) throws IOException { + String suffix = ".lock"; + // delete all directories without a lock file + FileFilter filter = p -> p.getName().startsWith(name) && !p.getName().endsWith(suffix); + File[] dirs = parent.listFiles(filter); + if (dirs == null) + throw new IOException("I/O error or not a directory: " + parent); + + for (File dir : dirs) { + if (new File(dir + suffix).exists()) + continue; + deleteDir(dir); + } + + File tempDir = Files.createTempDirectory(parent.toPath(), name).toFile(); + File lock = new File(tempDir + suffix); + lock.createNewFile(); + lock.deleteOnExit(); + + return tempDir; + } + +} diff --git a/src/org/helioviewer/jhv/io/JSONUtils.java b/src/org/helioviewer/jhv/io/JSONUtils.java new file mode 100644 index 0000000..4b2ad63 --- /dev/null +++ b/src/org/helioviewer/jhv/io/JSONUtils.java @@ -0,0 +1,53 @@ +package org.helioviewer.jhv.io; + +import java.io.ByteArrayOutputStream; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; +import java.io.Reader; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.util.zip.GZIPOutputStream; + +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONTokener; + +public class JSONUtils { + + private static final int BUFSIZ = 65536; + + public static JSONObject get(InputStream in) throws IOException, JSONException { + try (BufferedReader reader = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8), BUFSIZ)) { + return new JSONObject(new JSONTokener(reader)); + } + } + + public static JSONObject get(Reader in) { + return new JSONObject(new JSONTokener(in)); + } + + public static JSONObject get(URI uri) throws IOException, JSONException { + try (NetClient nc = NetClient.of(uri)) { + return get(nc.getReader()); + } + } + + public static JSONObject get(String uri) throws IOException, JSONException { + try (NetClient nc = NetClient.of(uri)) { + return get(nc.getReader()); + } + } + + public static ByteArrayOutputStream compressJSON(JSONObject json) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (GZIPOutputStream gz = new GZIPOutputStream(baos, BUFSIZ); + OutputStreamWriter out = new OutputStreamWriter(gz, StandardCharsets.UTF_8)) { + json.write(out); + } + return baos; + } + +} diff --git a/src/org/helioviewer/jhv/io/NetClient.java b/src/org/helioviewer/jhv/io/NetClient.java new file mode 100644 index 0000000..c6f2345 --- /dev/null +++ b/src/org/helioviewer/jhv/io/NetClient.java @@ -0,0 +1,56 @@ +package org.helioviewer.jhv.io; + +import java.awt.EventQueue; + +import java.io.IOException; +import java.io.InputStream; +import java.io.Reader; +import java.net.URI; +import java.net.URISyntaxException; +import okio.BufferedSource; + +public interface NetClient extends AutoCloseable { + + boolean isSuccessful(); + InputStream getStream(); + Reader getReader(); + BufferedSource getSource(); + long getContentLength(); + @Override void close() throws IOException; + + enum NetCache { + CACHE, NETWORK, BYPASS + } + + static NetClient of(String uri) throws IOException { + try { + return of(new URI(uri), false, NetCache.CACHE); + } catch (URISyntaxException e) { + throw new IOException(e); + } + } + + static NetClient of(URI uri) throws IOException { + return of(uri, false, NetCache.CACHE); + } + + static NetClient of(String uri, boolean allowError) throws IOException { + try { + return of(new URI(uri), allowError, NetCache.CACHE); + } catch (URISyntaxException e) { + throw new IOException(e); + } + } + + static NetClient of(URI uri, boolean allowError) throws IOException { + return of(uri, allowError, NetCache.CACHE); + } + + static NetClient of(URI uri, boolean allowError, NetCache cache) throws IOException { + if (EventQueue.isDispatchThread()) + throw new IOException("Don't do that"); + + return "file".equals(uri.getScheme()) ? new NetClientLocal(uri) : new NetClientRemote(uri, allowError, cache); + } + +} diff --git a/src/org/helioviewer/jhv/io/NetClientLocal.java b/src/org/helioviewer/jhv/io/NetClientLocal.java new file mode 100644 index 0000000..21e4cde --- /dev/null +++ b/src/org/helioviewer/jhv/io/NetClientLocal.java @@ -0,0 +1,56 @@ +package org.helioviewer.jhv.io; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Reader; +import java.net.URI; +import java.nio.charset.StandardCharsets; + +import okio.BufferedSource; +import okio.Okio; + +class NetClientLocal implements NetClient { + + private final BufferedSource response; + + NetClientLocal(URI uri) throws IOException { + try { + response = Okio.buffer(Okio.source(new File(uri.getPath()))); + } catch (Exception e) { + throw new IOException(e); + } + } + + @Override + public boolean isSuccessful() { + return true; + } + + @Override + public InputStream getStream() { + return response.inputStream(); + } + + @Override + public Reader getReader() { + return new InputStreamReader(response.inputStream(), StandardCharsets.UTF_8); + } + + @Override + public BufferedSource getSource() { + return response; + } + + @Override + public long getContentLength() { + return -1; + } + + @Override + public void close() throws IOException { + response.close(); + } + +} diff --git a/src/org/helioviewer/jhv/io/NetClientRemote.java b/src/org/helioviewer/jhv/io/NetClientRemote.java new file mode 100644 index 0000000..e67fc1b --- /dev/null +++ b/src/org/helioviewer/jhv/io/NetClientRemote.java @@ -0,0 +1,107 @@ +package org.helioviewer.jhv.io; + +import java.io.IOException; +import java.io.InputStream; +import java.io.Reader; +import java.net.URI; +import java.util.concurrent.TimeUnit; + +//import java.util.logging.Level; +//import java.util.logging.Logger; + +import javax.annotation.Nonnull; + +import org.helioviewer.jhv.JHVGlobals; +import org.helioviewer.jhv.log.Log; + +import okhttp3.Cache; +import okhttp3.CacheControl; +import okhttp3.HttpUrl; +import okhttp3.Interceptor; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.Response; +import okio.BufferedSource; + +class NetClientRemote implements NetClient { + + private static final int cacheSize = 512 * 1024 * 1024; + private static final CacheControl noStore = new CacheControl.Builder().noStore().build(); + private static final OkHttpClient client = new OkHttpClient.Builder() + .connectTimeout(JHVGlobals.getStdConnectTimeout(), TimeUnit.MILLISECONDS) + .readTimeout(JHVGlobals.getStdReadTimeout(), TimeUnit.MILLISECONDS) + //.cache(new Cache(JHVGlobals.clientCacheDir, cacheSize)) + //.addInterceptor(new LoggingInterceptor()) + .build(); + + private final Response response; + + NetClientRemote(URI uri, boolean allowError, NetCache cache) throws IOException { + HttpUrl url = HttpUrl.get(uri); + if (url == null) + throw new IOException("Could not parse " + uri); + + Request.Builder builder = new Request.Builder().header("User-Agent", JHVGlobals.userAgent).url(url); + if (cache == NetCache.NETWORK) + builder.cacheControl(CacheControl.FORCE_NETWORK); + else if (cache == NetCache.BYPASS) + builder.cacheControl(noStore); + Request request = builder.build(); + //System.out.println(">>> " + url); + + response = client.newCall(request).execute(); + if (!allowError && !response.isSuccessful()) { + response.close(); + throw new IOException(response.toString()); + } + + //if (response.cacheResponse() != null) + // System.out.println(">>> cached response: " + url); + } + + @Override + public boolean isSuccessful() { + return response.isSuccessful(); + } + + @Override + public InputStream getStream() { + return response.body().byteStream(); + } + + @Override + public Reader getReader() { + return response.body().charStream(); + } + + @Override + public BufferedSource getSource() { + return response.body().source(); + } + + @Override + public long getContentLength() { + return response.body().contentLength(); + } + + @Override + public void close() { + response.close(); + } + + private static class LoggingInterceptor implements Interceptor { + @Override + public Response intercept(@Nonnull Chain chain) throws IOException { + long t1 = System.nanoTime(); + Request r1 = chain.request(); + Log.info(String.format("Sending request %s on %s%n%s", r1.url(), chain.connection(), r1.headers())); + + Response r2 = chain.proceed(r1); + long t2 = System.nanoTime(); + Log.info(String.format("Received response for %s in %.1fms%n%s", r1.url(), (t2 - t1) / 1e6d, r2.networkResponse().headers())); + + return r2; + } + } + +} diff --git a/src/org/helioviewer/jhv/log/Log.java b/src/org/helioviewer/jhv/log/Log.java index 490546d..f863047 100644 --- a/src/org/helioviewer/jhv/log/Log.java +++ b/src/org/helioviewer/jhv/log/Log.java @@ -8,6 +8,10 @@ private static String format(String s) { return TimeUtils.format(System.currentTimeMillis()) + " [" + Thread.currentThread().getName() + "] " + s; } + public static void info(Object obj) { + System.err.println(format("INFO " + obj.toString())); + } + public static void warn(Object obj) { System.err.println(format("WARN " + obj.toString())); } diff --git a/src/org/helioviewer/jhv/time/JHVDate.java b/src/org/helioviewer/jhv/time/JHVDate.java index 633ac92..2ebd3db 100644 --- a/src/org/helioviewer/jhv/time/JHVDate.java +++ b/src/org/helioviewer/jhv/time/JHVDate.java @@ -20,7 +20,7 @@ public JHVDate(long _milli) { @Override public int compareTo(@Nonnull JHVDate dt) { - return milli < dt.milli ? -1 : (milli > dt.milli ? +1 : 0); + return Long.compare(milli, dt.milli); } @Override diff --git a/src/org/helioviewer/jhv/time/TimeUtils.java b/src/org/helioviewer/jhv/time/TimeUtils.java index 774abb4..c4d7823 100644 --- a/src/org/helioviewer/jhv/time/TimeUtils.java +++ b/src/org/helioviewer/jhv/time/TimeUtils.java @@ -10,6 +10,7 @@ import java.util.Optional; import org.everit.json.schema.FormatValidator; +//import org.ocpsoft.prettytime.nlp.PrettyTimeParser; public class TimeUtils { @@ -17,13 +18,37 @@ public class TimeUtils { private static final DateTimeFormatter sqlFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); private static final DateTimeFormatter fileFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd_HH.mm.ss"); +// private static final PrettyTimeParser prettyParser = new PrettyTimeParser(); + public static final long DAY_IN_MILLIS = 86400000; public static final long MINUTE_IN_MILLIS = 60000; - public static final JHVDate EPOCH = new JHVDate("2000-01-01T00:00:00"); + public static final JHVDate START = new JHVDate(floorSec(System.currentTimeMillis())); public static final JHVDate MINIMAL_DATE = new JHVDate("1970-01-01T00:00:00"); public static final JHVDate MAXIMAL_DATE = new JHVDate("2050-01-01T00:00:00"); + private static final double MAX_FRAMES = 97; + + public static int defaultCadence(long start, long end) { + return (int) Math.max(1, (end - start) / MAX_FRAMES / 1000); + } + + public static long floorSec(long milli) { + return (milli / 1000L) * 1000L; + } + + public static long roundSec(long milli) { + return ((milli + 500L) / 1000L) * 1000L; + } + + public static long ceilSec(long milli) { + return ((milli + 999L) / 1000L) * 1000L; + } + + public static long floorDay(long milli) { + return milli - milli % DAY_IN_MILLIS; + } + public static String format(long milli) { return DateTimeFormatter.ISO_LOCAL_DATE_TIME.format(Instant.ofEpochMilli(milli).atOffset(ZERO)); } @@ -63,11 +88,19 @@ public static long parseDate(String date) { public static long parseTime(String date) { return LocalTime.parse(date, DateTimeFormatter.ISO_LOCAL_TIME).toSecondOfDay() * 1000L; } - +/* + public static long optParse(String date, long alt) { + try { + return roundSec(prettyParser.parse(date).get(0).getTime()); + } catch (Exception e) { + return alt; + } + } +*/ public static class SQLDateTimeFormatValidator implements FormatValidator { @Override - public Optional validate(final String subject) { + public Optional validate(String subject) { try { long time = parseSQL(subject); if (time < MINIMAL_DATE.milli || time > MAXIMAL_DATE.milli)