Skip to content

Commit

Permalink
Atlas single cell web cli (#222)
Browse files Browse the repository at this point in the history
* Atlas single cell web cli

* Fix cli test and remove more unnecessary things.

* Wrong deps

* CreateExperiment cli and handle failed accessions

* Log success with accessKeyGenerated

* Explains that create also updates.

* A bit of refactoring

* Review suggestions

* Remove unneeded tests that were just dragged.

* Pjn to web core with atlas-bioentities pointer

* Be able to give only private accessions if needed.

* Missing deps compared to web-bulk cli PR

* Most closely resemble PR ebi-gene-expression-group/atlas-web-bulk#88

* Latest web core and attempt profile !Cli for currently offending part

* More profile additions that end up producing more complicated errors.

* Remove any Spring dependencies from parent plug-in
We want app to include Spring framework and CLI to include Spring Boot

* Don’t include web config (i.e. ServletContext-dependent classes) in the CLI module

* Avoid null pointer exception by makings sure collections are not null.

Co-authored-by: Alfonso Muñoz-Pomer Fuentes <[email protected]>
  • Loading branch information
pcm32 and alfonsomunozpomer authored Feb 22, 2022
1 parent b23017d commit fa7a3a9
Show file tree
Hide file tree
Showing 22 changed files with 423 additions and 20 deletions.
25 changes: 25 additions & 0 deletions .github/workflows/cli-pr.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
name: Build atlas-web-single-cell cli
on:
pull_request

jobs:
compile_cli:
name: compile_cli
strategy:
fail-fast: false # if one of the jobs in the matrix expansion fails, the rest of the jobs will be cancelled
matrix:
os: ["ubuntu-latest"]
jdk: [11]
runs-on: ${{ matrix.os }}
env:
JDK_VERSION: ${{ matrix.jdk }}
steps:
- uses: actions/checkout@v2
with:
submodules: true
- uses: actions/setup-java@v1
with:
java-version: ${{ matrix.jdk }}
- name: Compile cli
run: ./gradlew --no-daemon :cli:bootJar

21 changes: 10 additions & 11 deletions app/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -22,27 +22,26 @@ def cache2kVersion = '1.2.4.Final'
dependencies {
implementation project(':atlas-web-core')

implementation 'org.springframework:spring-context:5.1.5.RELEASE'
implementation 'org.springframework:spring-web:5.1.5.RELEASE'
implementation 'org.springframework:spring-jdbc:5.1.5.RELEASE'
implementation 'org.springframework:spring-webmvc:5.1.5.RELEASE'

implementation 'javax.validation:validation-api:2.0.1.Final'
// Jackson BOM is included in 'atlas-web-app.java-conventions' so no need to specify version
implementation 'com.fasterxml.jackson.datatype:jackson-datatype-guava'
//implementation 'javax.validation:validation-api:2.0.1.Final'
runtimeOnly 'javax.validation:validation-api:2.0.1.Final'

implementation "org.cache2k:cache2k-api:${cache2kVersion}"
runtimeOnly "org.cache2k:cache2k-core:${cache2kVersion}"
implementation "org.cache2k:cache2k-spring:${cache2kVersion}"

//implementation 'org.apache.tiles:tiles-extras:3.0.8'
runtimeOnly 'org.apache.tiles:tiles-extras:3.0.8'
implementation 'javax.servlet.jsp:jsp-api:2.2'

// For URL rewrite (see WEB-INF/web.xml), or your Filter will fail with the most confusing error messages
//implementation 'org.tuckey:urlrewritefilter:4.0.3'
runtimeOnly 'org.tuckey:urlrewritefilter:4.0.3'
// implementation 'jstl:jstl:1.2'
runtimeOnly 'jstl:jstl:1.2'

testImplementation 'commons-beanutils:commons-beanutils:1.9.3'
testImplementation 'javax.servlet.jsp:jsp-api:2.2'

testImplementation 'com.zaxxer:HikariCP:3.3.1'
testImplementation 'org.junit.jupiter:junit-jupiter-api:5.4.1'
testImplementation 'org.junit.jupiter:junit-jupiter-engine:5.4.1'
testImplementation 'junit:junit:4.12'
testImplementation'org.junit.vintage:junit-vintage-engine:5.4.1'
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,12 @@

import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Profile;
import org.springframework.http.CacheControl;
import org.springframework.web.servlet.config.annotation.*;
import org.springframework.web.servlet.config.annotation.EnableWebMvc;
import org.springframework.web.servlet.config.annotation.InterceptorRegistry;
import org.springframework.web.servlet.config.annotation.ResourceHandlerRegistry;
import org.springframework.web.servlet.config.annotation.WebMvcConfigurer;
import org.springframework.web.servlet.view.UrlBasedViewResolver;
import org.springframework.web.servlet.view.tiles3.TilesConfigurer;
import org.springframework.web.servlet.view.tiles3.TilesView;
Expand All @@ -13,6 +17,7 @@
import javax.inject.Inject;
import java.util.concurrent.TimeUnit;

@Profile("!cli")
@Configuration
@EnableWebMvc
public class WebConfig implements WebMvcConfigurer {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.gson.JsonArray;
import org.springframework.context.annotation.Profile;
import org.springframework.http.MediaType;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
Expand All @@ -18,6 +19,7 @@
import static uk.ac.ebi.atlas.solr.bioentities.BioentityPropertyName.SYMBOL;
import static uk.ac.ebi.atlas.utils.GsonProvider.GSON;

@Profile("!cli")
@RestController
public class JsonBioentityInformationController extends JsonExceptionHandlingController {
private final BioEntityPropertyDao bioEntityPropertyDao;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package uk.ac.ebi.atlas.staticpages;

import org.springframework.context.annotation.Profile;
import org.springframework.core.io.Resource;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.PathVariable;
Expand All @@ -11,6 +12,7 @@
import javax.servlet.ServletContext;
import javax.servlet.http.HttpServletRequest;

@Profile("!cli")
@Controller
public class StaticPageController extends HtmlExceptionHandlingController {
private final ServletContextResourceLoader servletContextResourceLoader;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import org.junit.jupiter.api.TestInstance;
import org.junit.jupiter.api.extension.ExtendWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Profile;
import org.springframework.core.io.ClassPathResource;
import org.springframework.http.MediaType;
import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator;
Expand All @@ -33,6 +34,7 @@
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;

@ExtendWith(SpringExtension.class)
@Profile("!cli")
@WebAppConfiguration
@ContextConfiguration(classes = TestConfig.class)
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
Expand Down
2 changes: 1 addition & 1 deletion atlas-web-core
Submodule atlas-web-core updated 19 files
+7 −7 src/main/java/uk/ac/ebi/atlas/bioentity/properties/BioEntityCardProperties.java
+5 −1 src/main/java/uk/ac/ebi/atlas/bioentity/properties/BioEntityPropertyDao.java
+3 −2 src/main/java/uk/ac/ebi/atlas/configuration/JdbcConfig.java
+1 −1 src/main/java/uk/ac/ebi/atlas/configuration/SolrConfig.java
+3 −3 src/main/java/uk/ac/ebi/atlas/experimentpage/StaticFilesDownload.java
+14 −0 src/main/java/uk/ac/ebi/atlas/model/ExpressionUnitPropertyEditor.java
+11 −3 src/main/java/uk/ac/ebi/atlas/search/baseline/BaselineExperimentProfilesList.java
+33 −23 src/main/java/uk/ac/ebi/atlas/solr/analytics/query/AnalyticsQueryClient.java
+1 −1 src/main/java/uk/ac/ebi/atlas/solr/cloud/collections/BioentitiesCollectionProxy.java
+5 −4 src/main/java/uk/ac/ebi/atlas/testutils/JdbcUtils.java
+8 −0 src/main/java/uk/ac/ebi/atlas/web/ProteomicsDifferentialRequestPreferences.java
+1 −1 src/test/java/uk/ac/ebi/atlas/bioentity/properties/BioEntityPropertyDaoIT.java
+2 −2 src/test/java/uk/ac/ebi/atlas/experimentimport/idf/IdfParserIT.java
+2 −2 src/test/java/uk/ac/ebi/atlas/model/experiment/ExperimentConfigurationIT.java
+5 −5 src/test/java/uk/ac/ebi/atlas/resource/DataFileHubIT.java
+28 −17 src/test/java/uk/ac/ebi/atlas/search/baseline/BaselineExperimentProfilesListTest.java
+37 −36 src/test/java/uk/ac/ebi/atlas/species/SpeciesFactoryIT.java
+2 −2 src/test/java/uk/ac/ebi/atlas/trader/ConfigurationTraderIT.java
+2 −1 src/test/java/uk/ac/ebi/atlas/trader/ExperimentTraderDaoIT.java
Original file line number Diff line number Diff line change
Expand Up @@ -27,11 +27,6 @@ dependencies {
// Needs to match Tomcat version https://tomcat.apache.org/whichversion.html
implementation 'javax.servlet:javax.servlet-api:3.1.0'

implementation 'org.springframework:spring-context:5.1.5.RELEASE'
implementation 'org.springframework:spring-jdbc:5.1.5.RELEASE'
implementation 'org.springframework:spring-web:5.1.5.RELEASE'
implementation 'org.springframework:spring-webmvc:5.1.5.RELEASE'

// jackson-databind is required by RestTemplate included in Spring ¯\_(ツ)_/¯
implementation enforcedPlatform('com.fasterxml.jackson:jackson-bom:2.10.0')
implementation 'com.fasterxml.jackson.core:jackson-databind'
Expand All @@ -53,7 +48,7 @@ dependencies {
implementation 'com.google.code.gson:gson:2.8.5'
implementation 'com.jayway.jsonpath:json-path:2.4.0'

implementation 'org.apache.commons:commons-lang3:3.8.1'
implementation 'org.apache.commons:commons-lang3:3.12.0'
implementation 'org.apache.commons:commons-text:1.6'
implementation 'org.apache.commons:commons-math:2.2'
implementation 'commons-io:commons-io:2.10.0'
Expand Down
72 changes: 72 additions & 0 deletions cli/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
# Expression Atlas CLI Bulk
A minimal Spring Boot wrapper to run Single Cell Expression Atlas tasks from the command line.

## Requirements
- Java 11
- Expression Atlas environment (PostgreSQL server; SolrCloud cluster; bioentity annotation and experiment files)

## Usage
There are two main ways to run the application: as an executable JAR or via Gradle. The latter is recommended on
development environments and Java is preferred in production environments. Be aware that any changes made to the
properties file won’t take effect unless you rebuild the JAR file.

### Gradle
```bash
./gradlew :cli:bootRun --args="<task-name> <options>"
```

### Executable JAR
Build the JAR file:
```bash
./gradlew :cli:bootJar
```

Then run it with Java:
```bash
java -jar ./cli/build/libs/atlas-cli-sc.jar <task-name> <options>
```

## Configuration
Configuration variables are set with `-Dproperty=value` if you run the application via `java -jar ...`, or by adding
`-Pproperty=value` to the Gradle task (in the tables below: Java property name, and Gradle propery name, respectively).

**IMPORTANT**: At the very least you will need to set the environment variables described in the Default value columns
to run/compile the application with Gradle. However, notice that the `-D` arguments will override whatever was set at
compile time, so if you forget or your environment changes, you don’t need to recompile.

### Expression Atlas file options: `configuration.properties`
| Java property name | Gradle property name | Default value |
|-----------------------------|---------------------------|--------------------------|
| `data.files.location` | `dataFilesLocation` | `${ATLAS_DATA_PATH}` |
| `experiment.files.location` | `experimentFilesLocation` | `${ATLAS_DATA_PATH}/gxa` |

### Expression Atlas database options: `jdbc.properties`
| Java Property name | Gradle property name | Default value |
|--------------------|----------------------|---------------------------------------------------------------------|
| `jdbc.url` | `jdbcUrl` | `jdbc:postgresql://${ATLAS_POSTGRES_HOST}:5432/${ATLAS_POSTGRES_DB` |
| `jdbc.username` | `jdbcUsername` | `${ATLAS_POSTGRES_USER}` |
| `jdbc.password` | `jdbcPassword` | `${ATLAS_POSTRES_PASSWORD}` |

### Expression Atlas Solr options: `solr.properties`
| Java property name | Gradle property name | Default value |
|--------------------|----------------------|----------------------|
| `zk.host` | `zkHost` | `${ATLAS_ZK_HOST}` |
| `zk.port` | `zkPort` | `2181` |
| `solr.host` | `solrHost` | `${ATLAS_SOLR_HOST}` |
| `solr.port` | `solrPort` | `8983` |

## Tasks
Run without any arguments to get a list of available tasks:
```
Usage: <main class> [COMMAND]
Commands:
update-experiment-design Updates the experiment designs of an accession or a group of accessions.
```

Pass the name of a task to obtain a detailed description of available options:
```bash
$ java -jar ./cli/build/libs/atlas-cli-bulk.jar update-experiment-design
...

```

49 changes: 49 additions & 0 deletions cli/build.gradle
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
plugins {
id 'atlas-web-app.java-conventions'
id 'org.springframework.boot' version '2.4.2'
id 'io.spring.dependency-management' version '1.0.11.RELEASE'
}

version = '1.0.0'

bootJar {
archiveFileName = 'atlas-cli-sc.jar'
}

dependencies {
implementation project(':atlas-web-core')
implementation project(':app')

runtimeOnly 'javax.servlet.jsp:jsp-api:2.2'

implementation 'info.picocli:picocli:4.6.1'
implementation 'info.picocli:picocli-spring-boot-starter:4.6.1'

testImplementation 'org.springframework.boot:spring-boot-starter-test'
}

processResources {
filesMatching('configuration.properties') {
expand(
dataFilesLocation: project.hasProperty('dataFilesLocation') ? dataFilesLocation : "${System.env.ATLAS_DATA_PATH}",
experimentFilesLocation: project.hasProperty('experimentFilesLocation') ? experimentFilesLocation : "${System.env.ATLAS_DATA_PATH}/scxa",
)
}

filesMatching('jdbc.properties') {
expand(
jdbcUrl: project.hasProperty('jdbcUrl') ? jdbcUrl : "jdbc:postgresql://${System.env.ATLAS_POSTGRES_HOST}:5432/${System.env.ATLAS_POSTGRES_DB}",
jdbcUsername: project.hasProperty('jdbcUsername') ? jdbcUsername : "${System.env.ATLAS_POSTGRES_USER}",
jdbcPassword: project.hasProperty('jdbcPassword') ? jdbcPassword : "${System.env.ATLAS_POSTGRES_PASSWORD}"
)
}

filesMatching('solr.properties') {
expand(
zkHost: project.hasProperty('zkHost') ? zkHost : "${System.env.ATLAS_ZK_HOST}",
zkPort: project.hasProperty('zkPort') ? zkPort : '2181',
solrHost: project.hasProperty('solrHost') ? solrHost : "${System.env.ATLAS_SOLR_HOST}",
solrPort: project.hasProperty('solrPort') ? solrPort : '8983'
)
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
package uk.ac.ebi.atlas.cli;

import picocli.CommandLine;
import uk.ac.ebi.atlas.cli.utils.AccessionsWriter;

import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.logging.Logger;

public abstract class AbstractPerAccessionCommand {

private static final Logger LOGGER = Logger.getLogger(AbstractPerAccessionCommand.class.getName());;

@CommandLine.Option(names = {"-f", "--failed-accessions-path"}, description = "File to write failed accessions to.")
protected String failedOutputPath;

@CommandLine.Option(names = {"-e", "--experiment"}, split = ",", description = "one or more experiment accessions")
protected List<String> experimentAccessions = new ArrayList<>();

protected int handleFailedAccessions(Collection<String> failedAccessions) {
var status = 0;
if (!failedAccessions.isEmpty()) {
status = 1;
LOGGER.warning(String.format("%s experiments failed", failedAccessions.size()));
LOGGER.info(String.format("Re-run with the following arguments to re-try failed accessions: %s", String.join(",", failedAccessions)));
if (failedOutputPath != null) {
AccessionsWriter writer = new AccessionsWriter(failedOutputPath, failedAccessions);
writer.write();
}
}
return status;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
package uk.ac.ebi.atlas.cli;

import org.springframework.boot.CommandLineRunner;
import org.springframework.boot.ExitCodeGenerator;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import picocli.CommandLine;

@SpringBootApplication(scanBasePackages = "uk.ac.ebi.atlas")
public class ExpressionAtlasCliApplication implements CommandLineRunner, ExitCodeGenerator {
private final CommandLine.IFactory factory;
private final ExpressionAtlasCliCommand expressionAtlasCliCommand;

private int exitCode;

public ExpressionAtlasCliApplication(CommandLine.IFactory factory,
ExpressionAtlasCliCommand expressionAtlasCliCommand) {
this.factory = factory;
this.expressionAtlasCliCommand = expressionAtlasCliCommand;
}

public static void main(String[] args) {
System.exit(SpringApplication.exit(SpringApplication.run(ExpressionAtlasCliApplication.class, args)));
}

@Override
public void run(String... args) {
exitCode = new CommandLine(expressionAtlasCliCommand, factory).execute(args);
}

@Override
public int getExitCode() {
return exitCode;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
package uk.ac.ebi.atlas.cli;

import org.springframework.stereotype.Component;
import picocli.CommandLine.Command;
import uk.ac.ebi.atlas.cli.experiment.CreateUpdateExperimentCommand;
import uk.ac.ebi.atlas.cli.experiment.ExperimentDesignCommand;

@Command(subcommands = {
ExperimentDesignCommand.class,
CreateUpdateExperimentCommand.class
})
@Component
public class ExpressionAtlasCliCommand {
}
Loading

0 comments on commit fa7a3a9

Please sign in to comment.