From 8ba3e6094beea6f8d808b6c677eeb12034ae9cae Mon Sep 17 00:00:00 2001 From: Gavin Date: Mon, 29 Apr 2024 19:45:18 -0700 Subject: [PATCH] Use jitpacked jars --- .github/workflows/test.yml | 5 - README.md | 3 +- build.gradle | 27 +-- .../KBaseAuthenticatedFilterFactory.java | 3 +- .../assemblyhomology/service/api/Root.java | 2 +- .../test/assemblyhomology/TestCommon.java | 6 - .../workspace/WorkspaceController.java | 176 ------------------ .../integration/ServiceIntegrationTest.java | 7 +- .../service/api/RootTest.java | 2 +- .../controllers/workspace/wsjars | 37 ---- test.cfg.example | 3 - 11 files changed, 15 insertions(+), 256 deletions(-) delete mode 100644 src/test/java/us/kbase/test/assemblyhomology/controllers/workspace/WorkspaceController.java delete mode 100644 src/test/resources/us/kbase/test/assemblyhomology/controllers/workspace/wsjars diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5e6bfab..3a3ea1e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -46,10 +46,6 @@ jobs: # move to parent dir of homedir to install binaries etc cd .. - # set up jars - git clone https://github.com/kbase/jars - export JARSDIR=$(pwd)/jars/lib/jars/ - # Set up mash wget https://github.com/marbl/Mash/releases/download/v2.0/mash-Linux64-v2.0.tar tar -xf mash-Linux64-v2.0.tar @@ -66,7 +62,6 @@ jobs: # set up test config cd $HOMEDIR cp -n test.cfg.example test.cfg - sed -i "s#^test.jars.dir.*#test.jars.dir=$JARSDIR#" test.cfg sed -i "s#^test.mongo.exe.*#test.mongo.exe=$MONGOD#" test.cfg cat test.cfg diff --git a/README.md b/README.md index 9f18f2b..beea2c4 100644 --- a/README.md +++ b/README.md @@ -303,7 +303,8 @@ authorization source specified may be searched at the same time as namespaces wi * Releases * The master branch is the stable branch. Releases are made from the develop branch to the master branch. - * Update the version as per the semantic version rules in `src/us/kbase/assemblyhomoloy/api/Root.java`. + * Update the version as per the semantic version rules in + `src/us/kbase/assemblyhomoloy/service/api/Root.java`. * Tag the version in git and github. ### Running tests diff --git a/build.gradle b/build.gradle index 61e7507..688b658 100644 --- a/build.gradle +++ b/build.gradle @@ -9,8 +9,6 @@ plugins { id 'org.ajoberstar.grgit' version '4.1.1' } -// TODO NOW use jitpacked workspace client and shadow jar when available and remove jars - repositories { mavenCentral() @@ -111,26 +109,8 @@ task buildAll { dependsOn javadoc } -def fromURL = { url, name -> - File file = new File("$buildDir/download/${name}.jar") - file.parentFile.mkdirs() - if (!file.exists()) { - new URL(url).withInputStream { downloadStream -> - file.withOutputStream { fileOut -> - fileOut << downloadStream - } - } - } - files(file.absolutePath) -} - dependencies { - implementation fromURL( - 'https://github.com/kbase/jars/raw/master/lib/jars/kbase/workspace/WorkspaceService-0.8.0.jar', - 'WorkspaceService-0.8.0' - ) - implementation 'org.ini4j:ini4j:0.5.2' implementation 'commons-io:commons-io:2.4' implementation 'com.beust:jcommander:1.72' @@ -143,6 +123,12 @@ dependencies { exclude group: 'com.fasterxml.jackson.core' // breaks everything if we upgrade exclude group: 'javax.servlet', module: 'servlet-api' // 2.5 vs 3.1 below } + implementation('com.github.kbase.workspace_deluxe:workspace-client:0.15.0') { + exclude group: 'net.java.dev.jna' // breaks mac builds, not needed + exclude group: 'org.eclipse.jetty.aggregate' // ugh, java common pollutes everything + exclude group: 'com.fasterxml.jackson.core' // breaks everything if we upgrade + exclude group: 'javax.servlet', module: 'servlet-api' // 2.5 vs 3.0.1 below + } implementation 'com.github.zafarkhaja:java-semver:0.9.0' implementation 'org.yaml:snakeyaml:1.18' implementation 'ch.qos.logback:logback-classic:1.1.2' @@ -159,6 +145,7 @@ dependencies { implementation 'javax.xml.bind:jaxb-api:2.4.0-b180830.0359' testImplementation "com.github.kbase:auth2:0.7.1" + testImplementation 'com.github.kbase.workspace_deluxe:workspace_deluxe-test-shadow-all:0.15.0' testImplementation 'nl.jqno.equalsverifier:equalsverifier:3.1.10' testImplementation 'junit:junit:4.12' testImplementation('org.eclipse.jetty:jetty-servlet:9.3.11.v20160721') { diff --git a/src/main/java/us/kbase/assemblyhomology/filters/KBaseAuthenticatedFilterFactory.java b/src/main/java/us/kbase/assemblyhomology/filters/KBaseAuthenticatedFilterFactory.java index 82d95f9..e3972c0 100644 --- a/src/main/java/us/kbase/assemblyhomology/filters/KBaseAuthenticatedFilterFactory.java +++ b/src/main/java/us/kbase/assemblyhomology/filters/KBaseAuthenticatedFilterFactory.java @@ -170,8 +170,7 @@ public KBaseAuthenticatedFilter getFilter( if (e.getMessage().contains( // hacky hacky hacky // ws needs error codes - // and we need to rewrite the auth client for auth2 - "Login failed! Server responded with code 401 Unauthorized")) { + "10020 Invalid token")) { throw new MinHashDistanceFilterAuthenticationException("Invalid token"); } else { // this is annoying to test, so pass diff --git a/src/main/java/us/kbase/assemblyhomology/service/api/Root.java b/src/main/java/us/kbase/assemblyhomology/service/api/Root.java index fbc8f9b..a02c4bd 100644 --- a/src/main/java/us/kbase/assemblyhomology/service/api/Root.java +++ b/src/main/java/us/kbase/assemblyhomology/service/api/Root.java @@ -26,7 +26,7 @@ public class Root { //TODO ZLATER ROOT add configurable contact email or link //TODO ZLATER swagger - private static final String VERSION = "0.1.3"; + private static final String VERSION = "0.1.4"; private static final String SERVER_NAME = "Assembly Homology service"; /** Return the root information. diff --git a/src/test/java/us/kbase/test/assemblyhomology/TestCommon.java b/src/test/java/us/kbase/test/assemblyhomology/TestCommon.java index 5462889..a61c3f4 100644 --- a/src/test/java/us/kbase/test/assemblyhomology/TestCommon.java +++ b/src/test/java/us/kbase/test/assemblyhomology/TestCommon.java @@ -47,8 +47,6 @@ public class TestCommon { public static final String MONGOEXE = "test.mongo.exe"; public static final String MONGO_USE_WIRED_TIGER = "test.mongo.wired_tiger"; - public static final String JARS_PATH = "test.jars.dir"; - public static final String TEST_TEMP_DIR = "test.temp.dir"; public static final String KEEP_TEMP_DIR = "test.temp.dir.keep"; @@ -167,10 +165,6 @@ public static Path getMongoExe() { return Paths.get(getTestProperty(MONGOEXE)).toAbsolutePath().normalize(); } - public static Path getJarsDir() { - return Paths.get(getTestProperty(JARS_PATH)).toAbsolutePath().normalize(); - } - public static Path getTempDir() { return Paths.get(getTestProperty(TEST_TEMP_DIR)).toAbsolutePath().normalize(); } diff --git a/src/test/java/us/kbase/test/assemblyhomology/controllers/workspace/WorkspaceController.java b/src/test/java/us/kbase/test/assemblyhomology/controllers/workspace/WorkspaceController.java deleted file mode 100644 index b4d2eb0..0000000 --- a/src/test/java/us/kbase/test/assemblyhomology/controllers/workspace/WorkspaceController.java +++ /dev/null @@ -1,176 +0,0 @@ -package us.kbase.test.assemblyhomology.controllers.workspace; - -import static us.kbase.testutils.controllers.ControllerCommon.findFreePort; -import static us.kbase.testutils.controllers.ControllerCommon.makeTempDirs; - -import java.io.BufferedReader; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.Reader; -import java.net.URL; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Arrays; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Scanner; - -import org.apache.commons.io.FileUtils; -import org.bson.Document; -import org.ini4j.Ini; -import org.ini4j.Profile.Section; - -import com.mongodb.MongoClient; -import com.mongodb.client.MongoDatabase; - -import us.kbase.testutils.TestException; - - -/** Q&D Utility to run a Workspace server for the purposes of testing from - * Java. Ideally this'll be swapped out for a Docker container that runs automatically with - * kb-sdk test at some point. - * - * Initializes a GridFS backend and does not support handles. - * @author gaprice@lbl.gov - * - */ -public class WorkspaceController { - //TODO CODE move to workspace repo, follow auth controller pattern. Need some way of handling event listener configuration - - private final static String DATA_DIR = "temp_data"; - private static final String WS_CLASS = "us.kbase.workspace.WorkspaceServer"; - - private static final String JARS_FILE = "wsjars"; - - private final static List tempDirectories = new LinkedList(); - static { - tempDirectories.add(DATA_DIR); - } - - private final Path tempDir; - - private final Process workspace; - private final int port; - - public WorkspaceController( - final Path jarsDir, - final String mongoHost, - final String mongoDatabase, - final String adminUser, - final URL authServiceRootURL, - final Path rootTempDir) - throws Exception { - final String classpath = getClassPath(jarsDir); - tempDir = makeTempDirs(rootTempDir, "WorkspaceController-", tempDirectories); - port = findFreePort(); - final Path deployCfg = createDeployCfg( - mongoHost, mongoDatabase, authServiceRootURL, adminUser); - - try (final MongoClient mc = new MongoClient(mongoHost)) { - final MongoDatabase db = mc.getDatabase(mongoDatabase); - db.getCollection("settings").insertOne( - new Document("type_db", "WorkspaceController_types") - .append("backend", "gridFS")); - } - - final List command = new LinkedList(); - command.addAll(Arrays.asList("java", "-classpath", classpath, WS_CLASS, "" + port)); - final ProcessBuilder servpb = new ProcessBuilder(command) - .redirectErrorStream(true) - .redirectOutput(tempDir.resolve("workspace.log").toFile()); - - final Map env = servpb.environment(); - env.put("KB_DEPLOYMENT_CONFIG", deployCfg.toString()); - - workspace = servpb.start(); - Thread.sleep(5000); //wait for server to start up - } - - private Path createDeployCfg( - final String mongoHost, - final String mongoDatabase, - final URL authRootURL, - final String adminUser) - throws IOException { - final File iniFile = new File(tempDir.resolve("test.cfg").toString()); - System.out.println("Created temporary workspace config file: " + - iniFile.getAbsolutePath()); - final Ini ini = new Ini(); - Section ws = ini.add("Workspace"); - ws.add("mongodb-host", mongoHost); - ws.add("mongodb-database", mongoDatabase); - ws.add("auth-service-url", authRootURL.toString() + "/api/legacy/KBase"); - ws.add("auth-service-url-allow-insecure", "true"); - ws.add("globus-url", authRootURL.toString() + "/api/legacy/globus"); - ws.add("ws-admin", adminUser); - ws.add("temp-dir", tempDir.resolve("temp_data")); - ws.add("ignore-handle-service", "true"); - // search listener config - ini.store(iniFile); - return iniFile.toPath(); - } - - private String getClassPath(final Path jarsDir) - throws IOException { - final InputStream is = getClass().getResourceAsStream(JARS_FILE); - if (is == null) { - throw new TestException("No workspace versions file " + JARS_FILE); - } - final List classpath = new LinkedList<>(); - try (final Reader r = new InputStreamReader(is)) { - final BufferedReader br = new BufferedReader(r); - String line; - while ((line = br.readLine()) != null) { - if (!line.trim().isEmpty()) { - final Path jarPath = jarsDir.resolve(line); - if (Files.notExists(jarPath)) { - throw new TestException("Required jar does not exist: " + jarPath); - } - classpath.add(jarPath.toString()); - } - } - } - return String.join(":", classpath); - } - - public int getServerPort() { - return port; - } - - public Path getTempDir() { - return tempDir; - } - - public void destroy(boolean deleteTempFiles) throws IOException { - if (workspace != null) { - workspace.destroy(); - } - if (tempDir != null && deleteTempFiles) { - FileUtils.deleteDirectory(tempDir.toFile()); - } - } - - public static void main(String[] args) throws Exception { - WorkspaceController ac = new WorkspaceController( - Paths.get("/home/crusherofheads/localgit/jars"), - "localhost:27017", - "WSController", - "workspaceadmin", - new URL("https://ci.kbase.us/services/auth"), - Paths.get("workspacetesttemp")); - System.out.println(ac.getServerPort()); - System.out.println(ac.getTempDir()); - Scanner reader = new Scanner(System.in); - System.out.println("any char to shut down"); - //get user input for a - reader.next(); - ac.destroy(false); - reader.close(); - } - -} - diff --git a/src/test/java/us/kbase/test/assemblyhomology/integration/ServiceIntegrationTest.java b/src/test/java/us/kbase/test/assemblyhomology/integration/ServiceIntegrationTest.java index 8226ccf..f38fa77 100644 --- a/src/test/java/us/kbase/test/assemblyhomology/integration/ServiceIntegrationTest.java +++ b/src/test/java/us/kbase/test/assemblyhomology/integration/ServiceIntegrationTest.java @@ -74,11 +74,11 @@ import us.kbase.test.assemblyhomology.data.TestDataManager; import us.kbase.test.assemblyhomology.service.api.RootTest; import us.kbase.test.auth2.authcontroller.AuthController; +import us.kbase.test.workspace.controllers.workspace.WorkspaceController; import us.kbase.workspace.CreateWorkspaceParams; import us.kbase.workspace.SetPermissionsParams; import us.kbase.workspace.WorkspaceClient; import us.kbase.test.assemblyhomology.TestCommon; -import us.kbase.test.assemblyhomology.controllers.workspace.WorkspaceController; public class ServiceIntegrationTest { @@ -195,9 +195,9 @@ public static void beforeClass() throws Exception { // set up Workspace WS = new WorkspaceController( - TestCommon.getJarsDir(), "localhost:" + MANAGER.mongo.getServerPort(), "AssemblyHomologyServiceIntegTestWSDB", + "AssemblyHomologyServiceIntegTestWSDB_types", "fakeadmin", authURL, TEMP_DIR); @@ -485,8 +485,6 @@ public void searchNamespace() throws Exception { Files.newInputStream(TEMP_DIR.resolve(QUERY_K31_S1500)), MediaType.APPLICATION_OCTET_STREAM)); - assertThat("incorrect response code", res.getStatus(), is(200)); - @SuppressWarnings("unchecked") final Map response = res.readEntity(Map.class); @@ -515,6 +513,7 @@ public void searchNamespace() throws Exception { ); assertThat("incorrect response", response, is(expected)); + assertThat("incorrect response code", res.getStatus(), is(200)); } @Test diff --git a/src/test/java/us/kbase/test/assemblyhomology/service/api/RootTest.java b/src/test/java/us/kbase/test/assemblyhomology/service/api/RootTest.java index 1a3ad03..dd14fa7 100644 --- a/src/test/java/us/kbase/test/assemblyhomology/service/api/RootTest.java +++ b/src/test/java/us/kbase/test/assemblyhomology/service/api/RootTest.java @@ -17,7 +17,7 @@ public class RootTest { - public static final String SERVER_VER = "0.1.3"; + public static final String SERVER_VER = "0.1.4"; private static final String GIT_ERR = "Missing git commit file gitcommit, should be in us.kbase.assemblyhomology"; diff --git a/src/test/resources/us/kbase/test/assemblyhomology/controllers/workspace/wsjars b/src/test/resources/us/kbase/test/assemblyhomology/controllers/workspace/wsjars deleted file mode 100644 index 9606ac3..0000000 --- a/src/test/resources/us/kbase/test/assemblyhomology/controllers/workspace/wsjars +++ /dev/null @@ -1,37 +0,0 @@ -kbase/workspace/WorkspaceService-0.8.0.jar - -kbase/common/kbase-common-0.0.24.jar -ini4j/ini4j-0.5.2.jar -jetty/jetty-all-7.0.0.jar -jna/jna-3.4.0.jar -servlet/servlet-api-2.5.jar -syslog4j/syslog4j-0.9.46.jar -joda/joda-time-2.2.jar - -junit/junit-4.12.jar -hamcrest/hamcrest-core-1.3.jar -kbase/auth/kbase-auth-0.4.4.jar -jackson/jackson-annotations-2.2.3.jar -jackson/jackson-core-2.2.3.jar -jackson/jackson-databind-2.2.3.jar - -kbase/shock/shock-client-0.0.14.jar -apache_commons/commons-logging-1.1.1.jar -apache_commons/http/httpclient-4.3.1.jar -apache_commons/http/httpcore-4.3.jar -apache_commons/http/httpmime-4.3.1.jar - -kbase/kidl/kbase-kidl-parser-1409261812-7863aef.jar -apache_commons/commons-codec-1.8.jar -apache_commons/commons-io-2.4.jar -apache_commons/commons-lang3-3.1.jar -mongo/mongo-java-driver-2.13.3.jar -jongo/jongo-0.5-early-20130912-1506.jar -bson4jackson/bson4jackson-2.2.0-2.2.0.jar -slf4j/slf4j-api-1.7.7.jar -logback/logback-core-1.1.2.jar -logback/logback-classic-1.1.2.jar -google/guava-14.0.1.jar -kbase/handle/HandleServiceClient-160803-b9de699.jar -kbase/handle/HandleManagerClient-160803-08c8da3.jar - diff --git a/test.cfg.example b/test.cfg.example index a2daa58..f1480b3 100644 --- a/test.cfg.example +++ b/test.cfg.example @@ -6,9 +6,6 @@ # path to the mongodb executable to use for the tests. test.mongo.exe=/path/to/mongodbexecutable -# path to the KBase jars directory, e.g. [path to parent folder]/jars/lib/jars -test.jars.dir=/path/to/jars/dir - # true to use wired tiger, anything else for false. test.mongo.wired_tiger=false