Skip to content

Commit

Permalink
Merge pull request #122 from jgi-kbase/dev-gradle
Browse files Browse the repository at this point in the history
Add Gradle and update dockerfile / docker-compose
  • Loading branch information
MrCreosote authored May 1, 2024
2 parents fafcf72 + 422ec5f commit bde377b
Show file tree
Hide file tree
Showing 26 changed files with 551 additions and 407 deletions.
9 changes: 9 additions & 0 deletions .gitattributes
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
#
# https://help.github.com/articles/dealing-with-line-endings/
#
# Linux start script should use lf
/gradlew text eol=lf

# These are Windows script files and should use crlf
*.bat text eol=crlf

6 changes: 3 additions & 3 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,10 @@ jobs:
# the current production setup
- java: '8'
mongo: 'mongodb-linux-x86_64-3.6.13'
ant_test: 'test'
gradle_test: 'test'
- java: '11'
mongo: 'mongodb-linux-x86_64-3.6.23'
ant_test: 'test'
gradle_test: 'test'
steps:
- uses: actions/checkout@v3

Expand Down Expand Up @@ -75,7 +75,7 @@ jobs:
run: |
PATH=$PATH:$MASH_PATH
echo $PATH
ant ${{matrix.ant_test}}
./gradlew ${{matrix.gradle_test}}
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v3
Expand Down
30 changes: 22 additions & 8 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -1,10 +1,22 @@
FROM kbase/sdkbase2 as build
RUN cd / && git clone https://github.com/kbase/jars

WORKDIR /tmp/ah

ADD . /src
RUN cd /src && ant build
RUN find /src
# dependencies take a while to D/L, so D/L & cache before the build so code changes don't cause
# a new D/L
# can't glob *gradle because of the .gradle dir
COPY build.gradle gradlew settings.gradle /tmp/ah/
COPY gradle/ /tmp/ah/gradle/
RUN ./gradlew dependencies

# Now build the code
COPY deployment/ /tmp/ah/deployment/
COPY jettybase/ /tmp/ah/jettybase/
COPY src /tmp/ah/src/
COPY war /tmp/ah/war/
# for the git commit
COPY .git /tmp/ah/.git/
RUN ./gradlew war


FROM kbase/kb_jre:latest
Expand All @@ -14,10 +26,9 @@ ARG BUILD_DATE
ARG VCS_REF
ARG BRANCH=develop

COPY --from=build /src/deployment/ /kb/deployment/
COPY --from=build /src/jettybase/ /kb/deployment/jettybase/
COPY --from=build /src/dist/ /src/dist/
COPY --from=build /src/assembly_homology /kb/deployment/bin/
COPY --from=build /tmp/ah/deployment/ /kb/deployment/
COPY --from=build /tmp/ah/jettybase/ /kb/deployment/jettybase/
COPY --from=build /tmp/ah/build/libs/AssemblyHomologyService.war /kb/deployment/jettybase/webapps/ROOT.war

# The BUILD_DATE value seem to bust the docker cache when the timestamp changes, move to
# the end
Expand All @@ -33,6 +44,9 @@ ENV ASSEMBLY_HOMOLOGY_CONFIG=/kb/deployment/conf/deployment.cfg
ENV PATH=/bin:/usr/bin:/kb/deployment/bin
ENV JETTY_HOME=/usr/local/jetty

# TODO BUILD update to no longer use dockerize and take env vars (e.g. like Collections).
# TODO BUILD Use subsections in the ini file / switch to TOML

RUN wget https://github.com/marbl/Mash/releases/download/v2.0/mash-Linux64-v2.0.tar && \
tar xf mash-Linux64-v2.0.tar && \
cp mash-Linux64-v2.0/mash /usr/bin/mash
Expand Down
20 changes: 11 additions & 9 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,6 @@ This repo contains the KBase / JGI Assembly Homology Service (AHS). The service
assembly matching based on implementations of the
[MinHash algorithm](https://ieeexplore.ieee.org/abstract/document/666900/?reload=tru).

Build status (master):
[![Build Status](https://travis-ci.org/jgi-kbase/AssemblyHomologyService.svg?branch=master)](https://travis-ci.org/jgi-kbase/AssemblyHomologyService)
[![codecov](https://codecov.io/gh/jgi-kbase/AssemblyHomologyService/branch/master/graph/badge.svg)](https://codecov.io/gh/jgi-kbase/AssemblyHomologyService)

## Usage
Expand Down Expand Up @@ -33,21 +31,20 @@ is supported. Mash is configured to never return sequences with a distance great
## Requirements

Java 8 (OpenJDK OK)
Apache Ant (http://ant.apache.org/)
MongoDB 2.6+ (https://www.mongodb.com/)
Jetty 9.3+ (http://www.eclipse.org/jetty/download.html)
(see jetty-config.md for version used for testing)
This repo (git clone https://github.com/jgi-kbase/AssemblyHomologyService)
The jars repo (git clone https://github.com/kbase/jars)
The two repos above need to be in the same parent folder.

## Build

```
cd [assembly homology repo directory]
ant build
./gradlew buildAll
```

Build artifacts reside in the `build` directory.

## Load data

These instructions assume
Expand Down Expand Up @@ -230,7 +227,13 @@ processing and either discard or pass on all distances to the collector.
ensure `mash` is available on the system path
start mongodb
cd into the assembly homology repo
`ant build`

```
./gradlew buildAll
mkdir jettybase/webapps
cp build/libs/AssemblyHomologyService.war jettybase/webapps/ROOT.war
```

copy `deploy.cfg.example` to `deploy.cfg` and fill in appropriately
`export ASSEMBLY_HOMOLOGY_CONFIG=<path to deploy.cfg>`
OR
Expand Down Expand Up @@ -306,8 +309,7 @@ authorization source specified may be searched at the same time as namespaces wi
### Running tests

* Copy `test.cfg.example` to `test.cfg` and fill in the values appropriately.
* If it works as is start buying lottery tickets immediately.
* `ant test`
* `./gradlew test`

### UI

Expand Down
5 changes: 5 additions & 0 deletions RELEASE_NOTES.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,10 @@
# Assembly Homology Service release notes

## 0.1.4

* Switched the build system from Ant to Gradle. As such, all build artifacts, including the
`assembly_homology` script, are now found in the `build` directory.

## 0.1.3

* Update the Java Mongo client to 3.10.1. This supports the latest Mongo version, 4.0.
Expand Down
200 changes: 200 additions & 0 deletions build.gradle
Original file line number Diff line number Diff line change
@@ -0,0 +1,200 @@
/*
* This file was generated by the Gradle 'init' task.
*/

plugins {
id 'java'
id 'war'
id 'jacoco'
id 'org.ajoberstar.grgit' version '4.1.1'
}

// TODO NOW switch to std gradle layout
// TODO NOW use jitpacked workspace client and shadow jar when available and remove jars

repositories {
mavenCentral()

maven {
name = "Clojars"
url = "https://repo.clojars.org/"
}
maven {
name = "JitPack"
url = 'https://jitpack.io'
}
}
task buildGitCommitFile {
doLast {
def commitId = grgit.head().id
// is there a variable for builddir/classes/java/main?
file("$buildDir/classes/java/main/us/kbase/assemblyhomology/gitcommit").text = commitId
}
}

compileJava {
// TODO BUILD remove when we no longer support java 8, use `options.release = 11` if needed
java.sourceCompatibility = JavaVersion.VERSION_1_8
java.targetCompatibility = JavaVersion.VERSION_1_8
finalizedBy buildGitCommitFile
}

test {
/*
* TODO TEST Figure out why tests fail without this and remove. Might have something to do
* with the stfuLoggers() call in many of the tests, might kill logging for tests that
* require it
* Although it seems to make Mongo start up correctly as well which is odd
*/
forkEvery = 1
/*
* TODO TEST split tests into mongo wrapper tests & all other tests (incl. integration).
* Set up GHA to run the non-mongo tests with a single version of mongo and run the
* mongo tests with matrixed mongo versions. Combine coverage at the end somehow
*/
systemProperty "ASSEMHOMOL_TEST_CONFIG", "./test.cfg"
testLogging {
exceptionFormat = 'full'
showStandardStreams = true
}
finalizedBy jacocoTestReport
}

// TODO TEST add a test that starts the server in a docker container and checks some simple cmds

jacocoTestReport {
reports {
xml.required = true
csv.required = true
}
}

javadoc {
options {
// I don't know why this isn't working, but it's not worth spending time on right now
links "https://docs.oracle.com/javase/8/docs/api/"
}
}

war {
webXml = file('war/web.xml')
}

configurations {
// can't directly access testImplementation, so extend and access
testimpl.extendsFrom testImplementation
}

task generateCLIScript {
dependsOn compileJava
doLast {
def dependencies = configurations.runtimeClasspath.collect { File file ->
file.absolutePath
}

def classpath = dependencies.join(':')

def scriptContent = """#!/bin/sh
CLASSPATH=$classpath
java -cp build/classes/java/main:\$CLASSPATH us.kbase.assemblyhomology.cli.AssemblyHomologyCLI \$@
"""

file("$buildDir/assembly_homology").text = scriptContent
file("$buildDir/assembly_homology").setExecutable(true)
}
}

task buildAll {
dependsOn generateCLIScript
dependsOn war
dependsOn javadoc
}

sourceSets {
main {
java {
srcDirs = ["src"]
exclude '**/test/**'
}
}
test {
java {
srcDirs = ["src"]
include '**/test/**'
}
resources {
srcDirs = ["src"]
include "**/*.msh"
include "**/*.txt"
include "**/wsjars"
}
}
}

def fromURL = { url, name ->
File file = new File("$buildDir/download/${name}.jar")
file.parentFile.mkdirs()
if (!file.exists()) {
new URL(url).withInputStream { downloadStream ->
file.withOutputStream { fileOut ->
fileOut << downloadStream
}
}
}
files(file.absolutePath)
}

dependencies {

implementation fromURL(
'https://github.com/kbase/jars/raw/master/lib/jars/kbase/workspace/WorkspaceService-0.8.0.jar',
'WorkspaceService-0.8.0'
)

implementation 'org.ini4j:ini4j:0.5.2'
implementation 'commons-io:commons-io:2.4'
implementation 'com.beust:jcommander:1.72'
implementation 'org.mongodb:mongo-java-driver:3.10.1'
implementation 'com.google.guava:guava:18.0'
implementation "com.github.kbase:auth2_client_java:0.5.0"
implementation('com.github.kbase:java_common:0.3.0') {
exclude group: 'net.java.dev.jna' // breaks mac builds, not needed
exclude group: 'org.eclipse.jetty.aggregate' // ugh, java common pollutes everything
exclude group: 'com.fasterxml.jackson.core' // breaks everything if we upgrade
exclude group: 'javax.servlet', module: 'servlet-api' // 2.5 vs 3.1 below
}
implementation 'com.github.zafarkhaja:java-semver:0.9.0'
implementation 'org.yaml:snakeyaml:1.18'
implementation 'ch.qos.logback:logback-classic:1.1.2'
implementation 'org.slf4j:slf4j-api:1.7.25'
// TODO DEPS Need to rework the java common logger to not use syslog4j at all since it's
// abandonware and has a ton of CVEs, even in the newer versions.
implementation "org.syslog4j:syslog4j:0.9.46"
implementation 'com.fasterxml.jackson.core:jackson-annotations:2.5.4'
implementation 'com.fasterxml.jackson.core:jackson-databind:2.5.4'
implementation 'com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:2.5.4'
implementation 'org.glassfish.jersey.containers:jersey-container-servlet:2.23.2'
implementation 'org.glassfish.jersey.media:jersey-media-json-jackson:2.23.2'
implementation 'javax.servlet:javax.servlet-api:3.0.1'
implementation 'javax.xml.bind:jaxb-api:2.4.0-b180830.0359'

testImplementation "com.github.kbase:auth2:0.7.1"
testImplementation 'nl.jqno.equalsverifier:equalsverifier:3.1.10'
testImplementation 'junit:junit:4.12'
testImplementation('org.eclipse.jetty:jetty-servlet:9.3.11.v20160721') {
exclude group: "javax.servlet", module: "javax.servlet-api" // upgrading breaks tests
}
testImplementation 'org.mockito:mockito-core:3.0.0'
testImplementation 'org.apache.commons:commons-lang3:3.5'
testImplementation('com.github.kbase:java_test_utilities:0.1.0') {
exclude group: 'com.fasterxml.jackson.core' // upgrading breaks stuff
}
}

task showTestClassPath {
doLast {
configurations.testimpl.each { println it }
}
}
Loading

0 comments on commit bde377b

Please sign in to comment.