Merge pull request #162 from umccr/config-refactor

feat: additional features to make configuration simpler

.github/workflows/tests.yml

@@ -61,8 +61,13 @@ jobs:
         with:
           command: build
           args: --all-targets --features s3-storage
-      - name: Run cargo test
+      - name: Run cargo tests
         uses: actions-rs/cargo@v1
         with:
           command: test
           args: --all-features
+      - name: Run cargo tests no default features
+        uses: actions-rs/cargo@v1
+        with:
+          command: test
+          args: --no-default-features

Cargo.toml

@@ -5,5 +5,5 @@ members = [
     "htsget-http",
     "htsget-lambda",
     "htsget-search",
-    "htsget-test",
-]
+    "htsget-test"
+]

data/events/event_get.json

@@ -1,6 +1,6 @@
 {
   "httpMethod": "GET",
-  "path": "/variants/vcf/sample1-bcbio-cancer",
+  "path": "/variants/1-vcf/sample1-bcbio-cancer",
   "body": null,
 
   "resource": "/{proxy+}",

data/events/event_parameterized_get.json

@@ -1,6 +1,6 @@
 {
   "httpMethod": "GET",
-  "path": "/variants/vcf/sample1-bcbio-cancer",
+  "path": "/variants/1-vcf/sample1-bcbio-cancer",
   "body": null,
 
   "resource": "/{proxy+}",

data/events/event_parameterized_post.json

@@ -1,6 +1,6 @@
 {
   "httpMethod": "POST",
-  "path": "/variants/vcf/sample1-bcbio-cancer",
+  "path": "/variants/1-vcf/sample1-bcbio-cancer",
   "body": "{\"format\": \"VCF\", \"regions\": [{\"referenceName\": \"chrM\"}]}",
 
   "resource": "/{proxy+}",

data/events/event_parameterized_post_class_header.json

@@ -1,6 +1,6 @@
 {
   "httpMethod": "POST",
-  "path": "/variants/vcf/sample1-bcbio-cancer",
+  "path": "/variants/1-vcf/sample1-bcbio-cancer",
   "body": "{\"format\": \"VCF\", \"class\": \"header\", \"regions\": [{\"referenceName\": \"chrM\"}]}",
 
   "resource": "/{proxy+}",

data/events/event_post.json

@@ -1,6 +1,6 @@
 {
   "httpMethod": "POST",
-  "path": "/variants/vcf/sample1-bcbio-cancer",
+  "path": "/variants/1-vcf/sample1-bcbio-cancer",
   "body": null,
 
   "resource": "/{proxy+}",

deploy/config/config_dev_umccr.toml

@@ -24,36 +24,29 @@ documentation_url = "https://github.com/umccr/htsget-rs"
 environment = "dev"
 
 [[resolvers]]
-regex = '^umccr-10c-data-dev/(?P<key>.*)$'
+regex = '^(umccr-10c-data-dev)/(?P<key>.*)$'
 substitution_string = '$key'
-storage_type = { type = 'S3', bucket = 'umccr-10c-data-dev' }
 
 [[resolvers]]
-regex = '^umccr-10f-data-dev/(?P<key>.*)$'
+regex = '^(umccr-10f-data-dev)/(?P<key>.*)$'
 substitution_string = '$key'
-storage_type = { type = 'S3', bucket = 'umccr-10f-data-dev' }
 
 [[resolvers]]
-regex = '^umccr-10g-data-dev/(?P<key>.*)$'
+regex = '^(umccr-10g-data-dev)/(?P<key>.*)$'
 substitution_string = '$key'
-storage_type = { type = 'S3', bucket = 'umccr-10g-data-dev' }
 
 [[resolvers]]
-regex = '^umccr-agha-test-dev/(?P<key>.*)$'
+regex = '^(umccr-agha-test-dev)/(?P<key>.*)$'
 substitution_string = '$key'
-storage_type = { type = 'S3', bucket = 'umccr-agha-test-dev' }
 
 [[resolvers]]
-regex = '^umccr-research-dev/(?P<key>.*)$'
+regex = '^(umccr-research-dev)/(?P<key>.*)$'
 substitution_string = '$key'
-storage_type = { type = 'S3', bucket = 'umccr-research-dev' }
 
 [[resolvers]]
-regex = '^umccr-primary-data-dev/(?P<key>.*)$'
+regex = '^(umccr-primary-data-dev)/(?P<key>.*)$'
 substitution_string = '$key'
-storage_type = { type = 'S3', bucket = 'umccr-primary-data-dev' }
 
 [[resolvers]]
-regex = '^umccr-validation-prod/(?P<key>.*)$'
+regex = '^(umccr-validation-prod)/(?P<key>.*)$'
 substitution_string = '$key'
-storage_type = { type = 'S3', bucket = 'umccr-validation-prod' }

deploy/config/config_prod_umccr.toml

@@ -20,16 +20,13 @@ documentation_url = "https://github.com/umccr/htsget-rs"
 environment = "prod"
 
 [[resolvers]]
-regex = '^umccr-research-dev/(?P<key>.*)$'
+regex = '^(umccr-research-dev)/(?P<key>.*)$'
 substitution_string = '$key'
-storage_type = { type = 'S3', bucket = 'umccr-research-dev' }
 
 [[resolvers]]
-regex = '^umccr-validation-prod/(?P<key>.*)$'
+regex = '^(umccr-validation-prod)/(?P<key>.*)$'
 substitution_string = '$key'
-storage_type = { type = 'S3', bucket = 'umccr-validation-prod' }
 
 [[resolvers]]
-regex = '^umccr-primary-data-prod/(?P<key>.*)$'
+regex = '^(umccr-primary-data-prod)/(?P<key>.*)$'
 substitution_string = '$key'
-storage_type = { type = 'S3', bucket = 'umccr-primary-data-prod' }

htsget-actix/README.md

@@ -29,16 +29,16 @@ This crate uses [htsget-config] for configuration. See [htsget-config] for detai
 
 To run an instance of this crate, execute the following command:
 ```sh
-cargo run -p htsget-actix
+cargo run -p htsget-actix --no-default-features
 ```
 Using the default configuration, this will start a ticket server on `127.0.0.1:8080` and a data block server on `127.0.0.1:8081`
 with data accessible from the [`data`][data] directory.
 
-If only `LocalStorage` is required, compiling code related `AwsS3Storage` can be avoided by running the following:
-
+To use `AwsS3Storage`, avoid compiling with no default features:
 ```sh
-cargo run -p htsget-actix --no-default-features
+cargo run -p htsget-actix
 ```
+This will start a ticket server with `AwsS3Storage` using a bucket called `"data"`.
 
 See [htsget-search] for details on how to structure files.
 
@@ -70,25 +70,25 @@ Some example requests using `curl` are shown below:
 * GET
 
 ```sh
-curl '127.0.0.1:8080/variants/vcf/sample1-bcbio-cancer'
+curl '127.0.0.1:8080/variants/data/vcf/sample1-bcbio-cancer'
 ```
 
 * POST
 
 ```sh
-curl --header "Content-Type: application/json" -d '{}' '127.0.0.1:8080/variants/vcf/sample1-bcbio-cancer'
+curl --header "Content-Type: application/json" -d '{}' '127.0.0.1:8080/variants/data/vcf/sample1-bcbio-cancer'
 ```
 
 * Parametrised GET
 
 ```sh
-curl '127.0.0.1:8080/variants/vcf/sample1-bcbio-cancer?format=VCF&class=header'
+curl '127.0.0.1:8080/variants/data/vcf/sample1-bcbio-cancer?format=VCF&class=header'
 ```
 
 * Parametrised POST
 
 ```sh
-curl --header "Content-Type: application/json" -d '{"format": "VCF", "regions": [{"referenceName": "chrM"}]}' '127.0.0.1:8080/variants/vcf/sample1-bcbio-cancer'
+curl --header "Content-Type: application/json" -d '{"format": "VCF", "regions": [{"referenceName": "chrM"}]}' '127.0.0.1:8080/variants/data/vcf/sample1-bcbio-cancer'
 ```
 
 * Service info

htsget-actix/benches/request_benchmarks.rs

@@ -10,8 +10,8 @@ use reqwest::blocking::ClientBuilder;
 use reqwest::Result;
 use serde::{Deserialize, Serialize};
 
+use htsget_config::types::{Headers, JsonResponse};
 use htsget_http::{PostRequest, Region};
-use htsget_search::htsget::{Headers, JsonResponse};
 use htsget_test::http_tests::{default_config_fixed_port, default_dir, default_dir_data};
 
 const REFSERVER_DOCKER_IMAGE: &str = "ga4gh/htsget-refserver:1.5.0";
@@ -54,6 +54,7 @@ fn request(url: reqwest::Url, json_content: &impl Serialize, client: &Client) ->
     .unwrap()
     .json()
     .unwrap();
+
   response
     .htsget
     .urls
@@ -144,6 +145,7 @@ fn start_htsget_rs() -> (DropGuard, String) {
     .arg("run")
     .arg("-p")
     .arg("htsget-actix")
+    .arg("--no-default-features")
     .env("HTSGET_PATH", default_dir_data())
     .env("RUST_LOG", "warn")
     .spawn()
@@ -234,7 +236,7 @@ fn criterion_benchmark(c: &mut Criterion) {
   bench_pair(
     &mut group,
     "[LIGHT] simple request",
-    format_url(&htsget_rs_url, "reads/bam/htsnexus_test_NA12878"),
+    format_url(&htsget_rs_url, "reads/data/bam/htsnexus_test_NA12878"),
     format_url(&htsget_refserver_url, "reads/htsnexus_test_NA12878"),
     &json_content,
   );
@@ -254,7 +256,7 @@ fn criterion_benchmark(c: &mut Criterion) {
   bench_pair(
     &mut group,
     "[LIGHT] with region",
-    format_url(&htsget_rs_url, "reads/bam/htsnexus_test_NA12878"),
+    format_url(&htsget_rs_url, "reads/data/bam/htsnexus_test_NA12878"),
     format_url(&htsget_refserver_url, "reads/htsnexus_test_NA12878"),
     &json_content,
   );
@@ -281,7 +283,7 @@ fn criterion_benchmark(c: &mut Criterion) {
   bench_pair(
     &mut group,
     "[LIGHT] with two regions",
-    format_url(&htsget_rs_url, "reads/bam/htsnexus_test_NA12878"),
+    format_url(&htsget_rs_url, "reads/data/bam/htsnexus_test_NA12878"),
     format_url(&htsget_refserver_url, "reads/htsnexus_test_NA12878"),
     &json_content,
   );
@@ -301,7 +303,7 @@ fn criterion_benchmark(c: &mut Criterion) {
   bench_pair(
     &mut group,
     "[LIGHT] with VCF",
-    format_url(&htsget_rs_url, "variants/vcf/sample1-bcbio-cancer"),
+    format_url(&htsget_rs_url, "variants/data/vcf/sample1-bcbio-cancer"),
     format_url(&htsget_refserver_url, "variants/sample1-bcbio-cancer"),
     &json_content,
   );
@@ -321,7 +323,10 @@ fn criterion_benchmark(c: &mut Criterion) {
   bench_pair(
     &mut group,
     "[HEAVY] with big VCF",
-    format_url(&htsget_rs_url, "variants/vcf/internationalgenomesample"),
+    format_url(
+      &htsget_rs_url,
+      "variants/data/vcf/internationalgenomesample",
+    ),
     format_url(&htsget_refserver_url, "variants/internationalgenomesample"),
     &json_content,
   );

htsget-actix/src/handlers/mod.rs

@@ -1,7 +1,7 @@
 use actix_web::{http::StatusCode, Either, Responder};
 
+use htsget_config::types::JsonResponse;
 use htsget_http::Result;
-use htsget_search::htsget::JsonResponse;
 use pretty_json::PrettyJson;
 
 pub use crate::handlers::service_info::{

htsget-actix/src/lib.rs

@@ -9,9 +9,7 @@ use tracing_actix_web::TracingLogger;
 
 use htsget_config::config::cors::CorsConfig;
 pub use htsget_config::config::{Config, DataServerConfig, ServiceInfo, TicketServerConfig, USAGE};
-#[cfg(feature = "s3-storage")]
-pub use htsget_config::regex_resolver::aws::S3Resolver;
-pub use htsget_config::regex_resolver::StorageType;
+pub use htsget_config::storage::Storage;
 use htsget_search::htsget::from_storage::HtsGetFromStorage;
 use htsget_search::htsget::HtsGet;
 use htsget_search::storage::local::LocalStorage;
@@ -144,10 +142,10 @@ mod tests {
   use actix_web::dev::ServiceResponse;
   use actix_web::{test, web, App};
   use async_trait::async_trait;
-  use htsget_search::htsget::JsonResponse;
   use tempfile::TempDir;
 
-  use htsget_search::storage::data_server::HttpTicketFormatter;
+  use htsget_config::types::JsonResponse;
+  use htsget_search::storage::data_server::BindDataServer;
   use htsget_test::http_tests::{config_with_tls, default_test_config};
   use htsget_test::http_tests::{
     Header as TestHeader, Response as TestResponse, TestRequest, TestServer,
@@ -197,6 +195,18 @@ mod tests {
 
   #[async_trait(?Send)]
   impl TestServer<ActixTestRequest<test::TestRequest>> for ActixTestServer {
+    async fn get_expected_path(&self) -> String {
+      let mut bind_data_server =
+        BindDataServer::try_from(self.get_config().data_server().clone()).unwrap();
+      let server = bind_data_server.bind_data_server().await.unwrap();
+      let addr = server.local_addr();
+
+      let path = self.get_config().data_server().local_path().to_path_buf();
+      tokio::spawn(async move { server.serve(path).await.unwrap() });
+
+      expected_url_path(self.get_config(), addr)
+    }
+
     fn get_config(&self) -> &Config {
       &self.config
     }
@@ -205,18 +215,13 @@ mod tests {
       ActixTestRequest(test::TestRequest::default())
     }
 
-    async fn test_server(&self, request: ActixTestRequest<test::TestRequest>) -> TestResponse {
-      let mut formatter =
-        HttpTicketFormatter::try_from(self.get_config().data_server().clone()).unwrap();
-      let server = formatter.bind_data_server().await.unwrap();
-      let addr = server.local_addr();
-
-      let path = self.get_config().data_server().local_path().to_path_buf();
-      tokio::spawn(async move { server.serve(path).await.unwrap() });
-
-      let expected_path = expected_url_path(self.get_config(), addr);
+    async fn test_server(
+      &self,
+      request: ActixTestRequest<test::TestRequest>,
+      expected_path: String,
+    ) -> TestResponse {
+      let response = self.get_response(request.0).await;
 
-      let response = self.get_response(request.0, formatter).await;
       let status: u16 = response.status().into();
       let mut headers = response.headers().clone();
       let bytes = test::read_body(response).await.to_vec();
@@ -243,7 +248,6 @@ mod tests {
     async fn get_response(
       &self,
       request: test::TestRequest,
-      _formatter: HttpTicketFormatter,
     ) -> ServiceResponse<EitherBody<BoxBody>> {
       println!("{:#?}", self.config);
       let app = test::init_service(

htsget-actix/src/main.rs

@@ -2,7 +2,7 @@ use tokio::select;
 
 use htsget_actix::run_server;
 use htsget_actix::Config;
-use htsget_search::storage::data_server::HttpTicketFormatter;
+use htsget_search::storage::data_server::BindDataServer;
 
 #[actix_web::main]
 async fn main() -> std::io::Result<()> {
@@ -13,9 +13,9 @@ async fn main() -> std::io::Result<()> {
 
     if config.data_server().enabled() {
       let server = config.data_server().clone();
-      let mut formatter = HttpTicketFormatter::from(server.clone());
+      let mut bind_data_server = BindDataServer::from(server.clone());
 
-      let local_server = formatter.bind_data_server().await?;
+      let local_server = bind_data_server.bind_data_server().await?;
       let local_server =
         tokio::spawn(async move { local_server.serve(&server.local_path()).await });
 

htsget-config/Cargo.toml

@@ -15,6 +15,8 @@ s3-storage = []
 default = ["s3-storage"]
 
 [dependencies]
+thiserror = "1.0"
+async-trait = "0.1"
 noodles = { version = "0.32", features = ["core"] }
 serde = { version = "1.0", features = ["derive"] }
 serde_with = "2.2"
@@ -30,3 +32,4 @@ http-serde = "1.1"
 
 [dev-dependencies]
 figment = { version = "0.10", features = ["test"] }
+tokio = { version = "1.25", features = ["macros", "rt-multi-thread"] }