From 517a5fdc8166e6307a53477c2a9e6bbd0d7bcac0 Mon Sep 17 00:00:00 2001 From: alex Date: Tue, 1 Oct 2019 22:30:24 -0700 Subject: [PATCH 1/3] add configuration for caching session --- README.md | 14 + .../plugins/sftp/AutoConfiguration.java | 8 +- .../plugins/sftp/SftpInputProperties.java | 257 ++++++++++-------- .../sftp/SftpInputPropertiesTests.java | 27 ++ 4 files changed, 190 insertions(+), 116 deletions(-) diff --git a/README.md b/README.md index b7476034..4e71c6d4 100644 --- a/README.md +++ b/README.md @@ -216,6 +216,8 @@ It support the [Common Options](#CommonOptions) and the following options: | [bcgov.access.input.sftp.allow-unknown-keys](#bcgovaccessinputsftpallow-unknown-keys) | boolean | No | | [bcgov.access.input.sftp.known-host-file](#bcgovaccessinputsftpknown-host-file) | String | Yes (if allow-unknown-key is false) | | [bcgov.access.input.sftp.server-alive-interval](#bcgovaccessinputsftpserver-alive-interval) | String | No | +| [bcgov.access.input.sftp.caching-session-wait-timeout](#bcgovaccessinputsftpcaching-session-wait-timeout) | Int | No | +| [bcgov.access.input.sftp.caching-session-max-pool-size](#bcgovaccessinputsft[caching-session-max-pool-size) | Int | No | ##### bcgov.access.input.sftp.host @@ -303,6 +305,18 @@ If allow-unknown-key is true, this property will be ignored. Sets the timeout interval (in milliseconds) before a server-alive message is sent, in case no message is received from the server. +##### bcgov.access.input.sftp.caching-session-wait-timeout + +* Value type is Integer + +Sets the limit of how long to wait for a session to become available. + +##### bcgov.access.input.sftp.caching-session-max-pool-size + +* Value type is Integer + +Modify the target session pool size; the actual pool size will adjust up/down to this size as and when sessions are requested or retrieved. + ## Output Plugins You can configure the document input using the `bcgov.access.output` property. diff --git a/jrcc-access-spring-boot-autoconfigure/src/main/java/ca/bc/gov/open/jrccaccess/autoconfigure/plugins/sftp/AutoConfiguration.java b/jrcc-access-spring-boot-autoconfigure/src/main/java/ca/bc/gov/open/jrccaccess/autoconfigure/plugins/sftp/AutoConfiguration.java index 5ef192aa..57b0b54a 100644 --- a/jrcc-access-spring-boot-autoconfigure/src/main/java/ca/bc/gov/open/jrccaccess/autoconfigure/plugins/sftp/AutoConfiguration.java +++ b/jrcc-access-spring-boot-autoconfigure/src/main/java/ca/bc/gov/open/jrccaccess/autoconfigure/plugins/sftp/AutoConfiguration.java @@ -83,7 +83,13 @@ public SessionFactory sftpSessionFactory() throws InvalidCo properties.getServerAliveInterval().ifPresent(serverAliveInterval -> factory.setServerAliveInterval(serverAliveInterval)); - return new CachingSessionFactory<>(factory); + + CachingSessionFactory cachingSessionFactory = new CachingSessionFactory<>(factory); + + this.properties.getCachingSessionMaxPoolSize().ifPresent(poolSize -> cachingSessionFactory.setPoolSize(poolSize)); + this.properties.getCachingSessionWaitTimeout().ifPresent(timeout -> cachingSessionFactory.setSessionWaitTimeout(timeout)); + + return cachingSessionFactory; } @Bean diff --git a/jrcc-access-spring-boot-autoconfigure/src/main/java/ca/bc/gov/open/jrccaccess/autoconfigure/plugins/sftp/SftpInputProperties.java b/jrcc-access-spring-boot-autoconfigure/src/main/java/ca/bc/gov/open/jrccaccess/autoconfigure/plugins/sftp/SftpInputProperties.java index 7ed4b6d1..48dbd821 100644 --- a/jrcc-access-spring-boot-autoconfigure/src/main/java/ca/bc/gov/open/jrccaccess/autoconfigure/plugins/sftp/SftpInputProperties.java +++ b/jrcc-access-spring-boot-autoconfigure/src/main/java/ca/bc/gov/open/jrccaccess/autoconfigure/plugins/sftp/SftpInputProperties.java @@ -10,160 +10,187 @@ /** * Represents the rabbitmq input plugin properties + * * @author alexjoybc * @since 0.6.0 - * */ @ConfigurationProperties(prefix = "bcgov.access.input.sftp") public class SftpInputProperties { - private String host; + private String host; + + @Min(0) + private Integer port; + + private String username; - @Min(0) - private Integer port; + private String password; - private String username; + private String remoteDirectory; - private String password; + private String filterPattern; - private String remoteDirectory; + private String cron; - private String filterPattern; + private String maxMessagePerPoll; - private String cron; + private String sshPrivateKey; - private String maxMessagePerPoll; + private String sshPrivatePassphrase; - private String sshPrivateKey; + private boolean allowUnknownKeys; - private String sshPrivatePassphrase; + private String knownHostFile; - private boolean allowUnknownKeys; + private Integer serverAliveInterval; - private String knownHostFile; + private Integer cachingSessionWaitTimeout; - private Integer serverAliveInterval; + private Integer cachingSessionMaxPoolSize; - public String getRemoteDirectory() { - return remoteDirectory; - } + public String getRemoteDirectory() { + return remoteDirectory; + } - public void setRemoteDirectory(String remoteDirectory) { - this.remoteDirectory = remoteDirectory; - } + public void setRemoteDirectory(String remoteDirectory) { + this.remoteDirectory = remoteDirectory; + } - public String getHost() { - return host == null ? "localhost" : host; - } + public String getHost() { + return host == null ? "localhost" : host; + } - public void setHost(String host) { - this.host = host; - } + public void setHost(String host) { + this.host = host; + } - public Integer getPort() { - return port == null ? 22 : port; - } + public Integer getPort() { + return port == null ? 22 : port; + } - public void setPort(String port) { - this.port = Integer.valueOf(port); - } + public void setPort(String port) { + this.port = Integer.valueOf(port); + } - public String getUsername() { - return username; - } + public String getUsername() { + return username; + } - public void setUsername(String username) { - this.username = username; - } + public void setUsername(String username) { + this.username = username; + } - public String getPassword() { - return password; - } + public String getPassword() { + return password; + } - public void setPassword(String password) { - this.password = password; - } + public void setPassword(String password) { + this.password = password; + } - public String getCron() { - return cron; - } + public String getCron() { + return cron; + } - public void setCron(String cron) { - this.cron = cron; - } + public void setCron(String cron) { + this.cron = cron; + } - public String getFilterPattern() { - return filterPattern == null ? "" : filterPattern ; - } + public String getFilterPattern() { + return filterPattern == null ? "" : filterPattern; + } - public void setFilterPattern(String filterPattern) { - this.filterPattern = filterPattern; - } + public void setFilterPattern(String filterPattern) { + this.filterPattern = filterPattern; + } - public String getMaxMessagePerPoll() { - return maxMessagePerPoll; - } + public String getMaxMessagePerPoll() { + return maxMessagePerPoll; + } - public void setMaxMessagePerPoll(String maxMessagePerPoll) { - this.maxMessagePerPoll = maxMessagePerPoll; - } + public void setMaxMessagePerPoll(String maxMessagePerPoll) { + this.maxMessagePerPoll = maxMessagePerPoll; + } - public Resource getSshPrivateKey() { + public Resource getSshPrivateKey() { - if(StringUtils.isBlank(this.sshPrivateKey)) return null; - return new ByteArrayResource(this.sshPrivateKey.getBytes()); - } + if (StringUtils.isBlank(this.sshPrivateKey)) return null; + return new ByteArrayResource(this.sshPrivateKey.getBytes()); + } - public void setSshPrivateKey(String sshPrivateKey) { - this.sshPrivateKey = sshPrivateKey; - } + public void setSshPrivateKey(String sshPrivateKey) { + this.sshPrivateKey = sshPrivateKey; + } - public String getSshPrivatePassphrase() { - return sshPrivatePassphrase; - } + public String getSshPrivatePassphrase() { + return sshPrivatePassphrase; + } - public void setSshPrivatePassphrase(String sshPrivatePassphrase) { - this.sshPrivatePassphrase = sshPrivatePassphrase; - } + public void setSshPrivatePassphrase(String sshPrivatePassphrase) { + this.sshPrivatePassphrase = sshPrivatePassphrase; + } - /** - * @return if allow Unknown Keys - */ - public boolean isAllowUnknownKeys() { - return this.allowUnknownKeys; - } + /** + * @return if allow Unknown Keys + */ + public boolean isAllowUnknownKeys() { + return this.allowUnknownKeys; + } + + /** + * Set to true to unconditionally allow connecting to an unknown host or when a host's key has changed (see knownHosts). + * Default false. Set to true if a knownHosts file is not provided. + * + * @param allowUnknowKeys : true or false + */ + public void setAllowUnknownKeys(boolean allowUnknowKeys) { + this.allowUnknownKeys = allowUnknowKeys; + } + + /** + * @return the filename that will be used for a host key repository. The file has the same format as OpenSSH's known_hosts file. + */ + public String getKnownHostFile() { + return this.knownHostFile; + } + + /** + * set the known_hosts file name, including path + * + * @param knownHostFile + */ + public void setKnownHostFile(String knownHostFile) { + this.knownHostFile = knownHostFile; + } + + + public Optional getServerAliveInterval() { + if (this.serverAliveInterval == null) return Optional.empty(); + return Optional.of(this.serverAliveInterval); + } + + public void setServerAliveInterval(String serverAliveInterval) { + this.serverAliveInterval = Integer.valueOf(serverAliveInterval); + } + + public Optional getCachingSessionWaitTimeout() { + return this.cachingSessionWaitTimeout == null ? + Optional.empty() : + Optional.of(this.cachingSessionWaitTimeout); + } + + public void setCachingSessionWaitTimeout(String CachingSessionWaitTimeout) { + this.cachingSessionWaitTimeout = Integer.valueOf(CachingSessionWaitTimeout); + } + + public Optional getCachingSessionMaxPoolSize() { + return this.cachingSessionMaxPoolSize == null ? + Optional.empty() : + Optional.of(this.cachingSessionMaxPoolSize); + } + + public void setCachingSessionMaxPoolSize(String cachingSessionMaxPoolSize) { + this.cachingSessionMaxPoolSize = Integer.valueOf(cachingSessionMaxPoolSize); + } - /** - * Set to true to unconditionally allow connecting to an unknown host or when a host's key has changed (see knownHosts). - * Default false. Set to true if a knownHosts file is not provided. - * @param allowUnknowKeys : true or false - */ - public void setAllowUnknownKeys(boolean allowUnknowKeys) { - this.allowUnknownKeys = allowUnknowKeys; - } - - /** - * @return the filename that will be used for a host key repository. The file has the same format as OpenSSH's known_hosts file. - */ - public String getKnownHostFile() { - return this.knownHostFile; - } - - /** - * set the known_hosts file name, including path - * @param knownHostFile - */ - public void setKnownHostFile(String knownHostFile) { - this.knownHostFile = knownHostFile; - } - - - public Optional getServerAliveInterval() { - if(this.serverAliveInterval == null) return Optional.empty(); - return Optional.of(this.serverAliveInterval); - } - - public void setServerAliveInterval(String serverAliveInterval) { - this.serverAliveInterval = Integer.valueOf(serverAliveInterval); - } } diff --git a/jrcc-access-spring-boot-autoconfigure/src/test/java/ca/bc/gov/open/jrccaccess/autoconfigure/plugins/sftp/SftpInputPropertiesTests.java b/jrcc-access-spring-boot-autoconfigure/src/test/java/ca/bc/gov/open/jrccaccess/autoconfigure/plugins/sftp/SftpInputPropertiesTests.java index 32ad5bf2..90e39ac3 100644 --- a/jrcc-access-spring-boot-autoconfigure/src/test/java/ca/bc/gov/open/jrccaccess/autoconfigure/plugins/sftp/SftpInputPropertiesTests.java +++ b/jrcc-access-spring-boot-autoconfigure/src/test/java/ca/bc/gov/open/jrccaccess/autoconfigure/plugins/sftp/SftpInputPropertiesTests.java @@ -138,4 +138,31 @@ public void not_set_server_alive_interval_should_succeed() { SftpInputProperties sut = new SftpInputProperties(); Assert.assertFalse(sut.getServerAliveInterval().isPresent()); } + + @Test + public void set_session_timeout_should_succeed() { + SftpInputProperties sut = new SftpInputProperties(); + sut.setCachingSessionWaitTimeout("20"); + Assert.assertEquals("20", sut.getCachingSessionWaitTimeout().get().toString()); + } + + @Test + public void session_timeout_not_set_should_succeed() { + SftpInputProperties sut = new SftpInputProperties(); + Assert.assertFalse(sut.getCachingSessionWaitTimeout().isPresent()); + } + + @Test + public void set_session_pool_size_should_succeed() { + SftpInputProperties sut = new SftpInputProperties(); + sut.setCachingSessionWaitTimeout("20"); + Assert.assertEquals("20", sut.getCachingSessionMaxPoolSize().get().toString()); + } + + @Test + public void session_pool_size_not_set_should_succeed() { + SftpInputProperties sut = new SftpInputProperties(); + Assert.assertFalse(sut.getCachingSessionMaxPoolSize().isPresent()); + } + } From 1b9f6ff1983f499d5e1aa1219ed57cb8fe3b6af5 Mon Sep 17 00:00:00 2001 From: alex Date: Tue, 1 Oct 2019 22:33:26 -0700 Subject: [PATCH 2/3] fixing issue with scope --- .../java/ca/bc/gov/open/jrccaccess/libs/StorageService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jrcc-document-access-libs/src/main/java/ca/bc/gov/open/jrccaccess/libs/StorageService.java b/jrcc-document-access-libs/src/main/java/ca/bc/gov/open/jrccaccess/libs/StorageService.java index 27775f92..fea4e25f 100644 --- a/jrcc-document-access-libs/src/main/java/ca/bc/gov/open/jrccaccess/libs/StorageService.java +++ b/jrcc-document-access-libs/src/main/java/ca/bc/gov/open/jrccaccess/libs/StorageService.java @@ -22,7 +22,7 @@ public interface StorageService { * @param digest MD5 Digest * @return Content from storage */ - public String getString(String key, String digest) throws DocumentMessageException; + String getString(String key, String digest) throws DocumentMessageException; /** * Delete a document from storage based on the key @@ -30,5 +30,5 @@ public interface StorageService { * @return bool * @throws DocumentMessageException */ - public Boolean deleteString(String key) throws DocumentMessageException; + Boolean deleteString(String key) throws DocumentMessageException; } From 77389220a857f43b96a2bbe9f54794147a800ea6 Mon Sep 17 00:00:00 2001 From: alex Date: Tue, 1 Oct 2019 22:33:38 -0700 Subject: [PATCH 3/3] fix unit test --- .../autoconfigure/plugins/sftp/SftpInputPropertiesTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jrcc-access-spring-boot-autoconfigure/src/test/java/ca/bc/gov/open/jrccaccess/autoconfigure/plugins/sftp/SftpInputPropertiesTests.java b/jrcc-access-spring-boot-autoconfigure/src/test/java/ca/bc/gov/open/jrccaccess/autoconfigure/plugins/sftp/SftpInputPropertiesTests.java index 90e39ac3..b5ac4a10 100644 --- a/jrcc-access-spring-boot-autoconfigure/src/test/java/ca/bc/gov/open/jrccaccess/autoconfigure/plugins/sftp/SftpInputPropertiesTests.java +++ b/jrcc-access-spring-boot-autoconfigure/src/test/java/ca/bc/gov/open/jrccaccess/autoconfigure/plugins/sftp/SftpInputPropertiesTests.java @@ -155,7 +155,7 @@ public void session_timeout_not_set_should_succeed() { @Test public void set_session_pool_size_should_succeed() { SftpInputProperties sut = new SftpInputProperties(); - sut.setCachingSessionWaitTimeout("20"); + sut.setCachingSessionMaxPoolSize("20"); Assert.assertEquals("20", sut.getCachingSessionMaxPoolSize().get().toString()); }