Add multi-tenant with-data image

kamu-data · Feb 29, 2024 · e29848c · e29848c
1 parent 246996b
commit e29848c
Show file tree

Hide file tree

Showing 9 changed files with 179 additions and 62 deletions.
diff --git a/.github/workflows/release-images.yaml b/.github/workflows/release-images.yaml
@@ -35,9 +35,11 @@ jobs:
         run: |
           cd images/
           make api-server-with-data
+          make api-server-with-data-mt
 
       - name: Publish images
         run: |
           cd images
           make api-server-push
           make api-server-with-data-push
+          make api-server-with-data-mt-push
diff --git a/Cross.toml b/Cross.toml
@@ -0,0 +1,8 @@
+# This avoids cross picking up custom linker from ~/.cargo/config.toml
+# See: https://github.com/cross-rs/cross/issues/621
+
+[build.env]
+passthrough = [
+    "CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=x86_64-linux-gnu-gcc",
+    "RUSTFLAGS",
+]
diff --git a/images/Makefile b/images/Makefile
@@ -3,6 +3,8 @@ KAMU_VERSION = $(shell cargo metadata --format-version 1 | jq -r '.packages[] |
 API_SERVER_VERSION = $(shell cargo metadata --format-version 1 | jq -r '.packages[] | select( .name == "kamu-api-server") | .version')
 
 
+#########################################################################################
+
 .PHONY: api-server
 api-server:
 	docker build \
@@ -19,6 +21,8 @@ api-server-push:
 	docker push $(IMAGE_REPO)/kamu-api-server:latest
 
 
+#########################################################################################
+
 .PHONY: api-server-with-data
 api-server-with-data:
 	docker build \
@@ -31,3 +35,19 @@ api-server-with-data:
 .PHONY: api-server-with-data-push
 api-server-with-data-push:
 	docker push $(IMAGE_REPO)/kamu-api-server:latest-with-data
+
+
+#########################################################################################
+
+.PHONY: api-server-with-data-mt
+api-server-with-data-mt:
+	docker build \
+		--build-arg KAMU_VERSION=latest-with-data-mt \
+		--build-arg API_SERVER_VERSION=$(API_SERVER_VERSION) \
+		-t $(IMAGE_REPO)/kamu-api-server:latest-with-data-mt \
+		api-server-with-data-mt/
+
+
+.PHONY: api-server-with-data-mt-push
+api-server-with-data-mt-push:
+	docker push $(IMAGE_REPO)/kamu-api-server:latest-with-data-mt
diff --git a/images/api-server-with-data-mt/Dockerfile b/images/api-server-with-data-mt/Dockerfile
@@ -0,0 +1,19 @@
+ARG KAMU_VERSION
+FROM ghcr.io/kamu-data/kamu-base:${KAMU_VERSION}
+ARG API_SERVER_VERSION
+
+WORKDIR /opt/kamu
+
+RUN wget -q https://github.com/kamu-data/kamu-platform/releases/download/v${API_SERVER_VERSION}/kamu-api-server-x86_64-unknown-linux-gnu.tar.gz && \
+    tar -xf kamu-api-server-x86_64-unknown-linux-gnu.tar.gz && \
+    chmod +x kamu-api-server-x86_64-unknown-linux-gnu/kamu-api-server && \
+    mv kamu-api-server-x86_64-unknown-linux-gnu/kamu-api-server /opt/kamu/ && \
+    rm -rf kamu-api-server-x86_64-unknown-linux-gnu*
+
+COPY config.yaml /opt/kamu/config.yaml
+
+ENTRYPOINT ["/usr/bin/tini", "--"]
+
+CMD [ "/opt/kamu/kamu-api-server", "--config=/opt/kamu/config.yaml", "--repo-url=file:///opt/kamu/workspace/.kamu/datasets", "--multi-tenant", "run", "--address=0.0.0.0", "--http-port=8080" ]
+
+EXPOSE 8080/tcp
diff --git a/images/api-server-with-data-mt/config.yaml b/images/api-server-with-data-mt/config.yaml
@@ -0,0 +1,81 @@
+auth:
+  providers:
+    # Dummy accounts for owners of example datasets
+    - kind: dummy
+      accounts:
+        - accountId: kamu
+          accountName: kamu
+          accountType: User
+          avatarUrl: https://avatars.githubusercontent.com/u/50896974?s=200&v=4
+          displayName: kamu
+        - accountId: sh101-bowen
+          accountName: sh101-bowen
+          accountType: User
+          avatarUrl: https://cdn-icons-png.flaticon.com/512/3118/3118054.png
+          displayName: sh101-bowen
+        - accountId: sh102-gambier
+          accountName: sh102-gambier
+          accountType: User
+          avatarUrl: https://cdn-icons-png.flaticon.com/512/3118/3118054.png
+          displayName: sh102-gambier
+        - accountId: sh103-howe
+          accountName: sh103-howe
+          accountType: User
+          avatarUrl: https://cdn-icons-png.flaticon.com/512/3118/3118054.png
+          displayName: sh103-howe
+        - accountId: sh104-granville
+          accountName: sh104-granville
+          accountType: User
+          avatarUrl: https://cdn-icons-png.flaticon.com/512/3118/3118054.png
+          displayName: sh104-granville
+        - accountId: sh105-keats
+          accountName: sh105-keats
+          accountType: User
+          avatarUrl: https://cdn-icons-png.flaticon.com/512/3118/3118054.png
+          displayName: sh105-keats
+        - accountId: sh106-seymour
+          accountName: sh106-seymour
+          accountType: User
+          avatarUrl: https://cdn-icons-png.flaticon.com/512/3118/3118054.png
+          displayName: sh106-seymour
+        - accountId: mim.dk
+          accountName: mim.dk
+          accountType: User
+          avatarUrl: https://cdn-icons-png.flaticon.com/512/3530/3530558.png
+          displayName: mim.dk
+        - accountId: mmo.gov.uk
+          accountName: mmo.gov.uk
+          accountType: User
+          avatarUrl: https://cdn-icons-png.flaticon.com/512/3530/3530558.png
+          displayName: mmo.gov.uk
+        - accountId: moa.gov.nl
+          accountName: moa.gov.nl
+          accountType: User
+          avatarUrl: https://cdn-icons-png.flaticon.com/512/3530/3530558.png
+          displayName: moa.gov.nl
+        - accountId: ofb.gouv.fr
+          accountName: ofb.gouv.fr
+          accountType: User
+          avatarUrl: https://cdn-icons-png.flaticon.com/512/3530/3530558.png
+          displayName: ofb.gouv.fr
+        - accountId: protectedseas.net
+          accountName: protectedseas.net
+          accountType: User
+          avatarUrl: https://cdn-icons-png.flaticon.com/512/3530/3530558.png
+          displayName: protectedseas.net
+        - accountId: acme.fishing.co
+          accountName: acme.fishing.co
+          accountType: User
+          avatarUrl: https://cdn-icons-png.flaticon.com/512/1090/1090630.png
+          displayName: ACME Fishing Company
+        - accountId: globalfishingwatch.org
+          accountName: globalfishingwatch.org
+          accountType: User
+          avatarUrl: https://cdn-icons-png.flaticon.com/512/744/744480.png
+          displayName: globalfishingwatch.org
+        - accountId: bmis-bycatch.org
+          accountName: bmis-bycatch.org
+          accountType: User
+          avatarUrl: https://cdn-icons-png.flaticon.com/128/10197/10197245.png
+          displayName: Bycatch Management Information System
+    - kind: github
diff --git a/images/api-server/Dockerfile b/images/api-server/Dockerfile
@@ -1,11 +1,9 @@
 ARG KAMU_VERSION
 FROM ghcr.io/kamu-data/kamu-base:${KAMU_VERSION}
-
 ARG API_SERVER_VERSION
 
 WORKDIR /opt/kamu
 
-# API Server
 RUN wget -q https://github.com/kamu-data/kamu-platform/releases/download/v${API_SERVER_VERSION}/kamu-api-server-x86_64-unknown-linux-gnu.tar.gz && \
     tar -xf kamu-api-server-x86_64-unknown-linux-gnu.tar.gz && \
     chmod +x kamu-api-server-x86_64-unknown-linux-gnu/kamu-api-server && \
@@ -15,7 +13,6 @@ RUN wget -q https://github.com/kamu-data/kamu-platform/releases/download/v${API_
 
 ENTRYPOINT ["/usr/bin/tini", "--"]
 
-# Local repository:
-CMD ["/opt/kamu/kamu-api-server --local-repo /opt/kamu/workspace run --address 0.0.0.0 --http-port 80"]
+CMD ["/opt/kamu/kamu-api-server", "--repo-url=file:///opt/kamu/workspace/.kamu/datasets", "run", "--address=0.0.0.0", "--http-port=8080"]
 
-EXPOSE 80/tcp
+EXPOSE 8080/tcp
diff --git a/src/app/api-server/src/app.rs b/src/app/api-server/src/app.rs
@@ -46,13 +46,16 @@ pub async fn run(matches: clap::ArgMatches) -> Result<(), InternalError> {
         Url::from_directory_path(workspace_dir.join("datasets")).unwrap()
     };
 
+    let multi_tenant = matches.get_flag("multi-tenant") || repo_url.scheme() != "file";
+
     let local_dir = tempfile::tempdir().unwrap();
 
     info!(
         version = VERSION,
         args = ?std::env::args().collect::<Vec<_>>(),
         repo_url = %repo_url,
         local_dir = %local_dir.path().display(),
+        config = ?config,
         "Initializing {}",
         BINARY_NAME
     );
@@ -63,10 +66,11 @@ pub async fn run(matches: clap::ArgMatches) -> Result<(), InternalError> {
             true,
         ),
         &repo_url,
+        multi_tenant,
     )
     .await;
 
-    let catalog = init_dependencies(config, &repo_url, local_dir.path())
+    let catalog = init_dependencies(config, &repo_url, multi_tenant, local_dir.path())
         .await
         .add_value(dependencies_graph_repository)
         .bind::<dyn kamu::domain::DependencyGraphRepository, kamu::DependencyGraphRepositoryInMemory>()
@@ -104,7 +108,7 @@ pub async fn run(matches: clap::ArgMatches) -> Result<(), InternalError> {
                 address,
                 sub.get_one("http-port").map(|p| *p),
                 catalog.clone(),
-                should_use_multi_tenancy(&repo_url),
+                multi_tenant,
             );
 
             let flightsql_server = crate::flightsql_server::FlightSqlServer::new(
@@ -210,39 +214,19 @@ pub fn load_config(path: Option<&PathBuf>) -> Result<ApiServerConfig, InternalEr
 
 /////////////////////////////////////////////////////////////////////////////////////////
 
+// TODO: Get rid of this
 pub async fn prepare_dependencies_graph_repository(
     current_account_subject: kamu::domain::CurrentAccountSubject,
     repo_url: &Url,
+    multi_tenant: bool,
 ) -> kamu::DependencyGraphRepositoryInMemory {
     // Construct a special catalog just to create 1 object, but with a repository
     // bound to API server command line user. It also should be authorized to access
     // any dataset.
 
     let mut b = CatalogBuilder::new();
 
-    match repo_url.scheme() {
-        "file" => {
-            let datasets_dir = repo_url.to_file_path().unwrap();
-
-            b.add_builder(
-                kamu::DatasetRepositoryLocalFs::builder()
-                    .with_root(datasets_dir.clone())
-                    .with_multi_tenant(false),
-            );
-            b.bind::<dyn kamu::domain::DatasetRepository, kamu::DatasetRepositoryLocalFs>();
-        }
-        "s3" | "s3+http" | "s3+https" => {
-            let s3_context = kamu::utils::s3_context::S3Context::from_url(&repo_url).await;
-
-            b.add_builder(
-                kamu::DatasetRepositoryS3::builder()
-                    .with_s3_context(s3_context.clone())
-                    .with_multi_tenant(true),
-            )
-            .bind::<dyn kamu::domain::DatasetRepository, kamu::DatasetRepositoryS3>();
-        }
-        _ => panic!("Unsupported repository scheme: {}", repo_url.scheme()),
-    }
+    configure_repository(&mut b, repo_url, multi_tenant).await;
 
     let special_catlaog = b
         .add::<event_bus::EventBus>()
@@ -260,6 +244,7 @@ pub async fn prepare_dependencies_graph_repository(
 pub async fn init_dependencies(
     config: ApiServerConfig,
     repo_url: &Url,
+    multi_tenant: bool,
     local_dir: &Path,
 ) -> CatalogBuilder {
     let mut b = dill::CatalogBuilder::new();
@@ -348,20 +333,46 @@ pub async fn init_dependencies(
     b.add::<kamu_adapter_auth_oso::OsoDatasetAuthorizer>();
     b.add::<kamu::domain::auth::DummyOdfServerAccessTokenResolver>();
 
+    configure_repository(&mut b, repo_url, multi_tenant).await;
+
+    if !multi_tenant {
+        b.add_value(DummyAuthProvider::new_with_default_account());
+        b.bind::<dyn kamu::domain::auth::AuthenticationProvider, DummyAuthProvider>();
+    } else {
+        // Default to GitHub auth
+        if config.auth.providers.is_empty() {
+            b.add::<kamu_adapter_oauth::OAuthGithub>();
+        }
+
+        for provider in config.auth.providers {
+            match provider {
+                AuthProviderConfig::Github(_) => {
+                    b.add::<kamu_adapter_oauth::OAuthGithub>();
+                }
+                AuthProviderConfig::Dummy(prov) => {
+                    b.add_value(DummyAuthProvider::new(prov.accounts));
+                    b.bind::<dyn kamu::domain::auth::AuthenticationProvider, DummyAuthProvider>();
+                }
+            }
+        }
+    }
+
+    b
+}
+
+async fn configure_repository(b: &mut CatalogBuilder, repo_url: &Url, multi_tenant: bool) {
     match repo_url.scheme() {
         "file" => {
             let datasets_dir = repo_url.to_file_path().unwrap();
 
             b.add_builder(
                 kamu::DatasetRepositoryLocalFs::builder()
                     .with_root(datasets_dir.clone())
-                    .with_multi_tenant(false),
+                    .with_multi_tenant(multi_tenant),
             );
             b.bind::<dyn kamu::domain::DatasetRepository, kamu::DatasetRepositoryLocalFs>();
 
             b.add::<kamu::ObjectStoreBuilderLocalFs>();
-            b.add_value(DummyAuthProvider::new_with_default_account());
-            b.bind::<dyn kamu::domain::auth::AuthenticationProvider, DummyAuthProvider>();
         }
         "s3" | "s3+http" | "s3+https" => {
             let s3_context = kamu::utils::s3_context::S3Context::from_url(&repo_url).await;
@@ -376,38 +387,9 @@ pub async fn init_dependencies(
             let allow_http = repo_url.scheme() == "s3+http";
             b.add_value(kamu::ObjectStoreBuilderS3::new(s3_context, allow_http))
                 .bind::<dyn kamu::domain::ObjectStoreBuilder, kamu::ObjectStoreBuilderS3>();
-
-            // Default to GitHub auth
-            if config.auth.providers.is_empty() {
-                b.add::<kamu_adapter_oauth::OAuthGithub>();
-            }
-
-            for provider in config.auth.providers {
-                match provider {
-                    AuthProviderConfig::Github(_) => {
-                        b.add::<kamu_adapter_oauth::OAuthGithub>();
-                    }
-                    AuthProviderConfig::Dummy(prov) => {
-                        b.add_value(DummyAuthProvider::new(prov.accounts));
-                        b.bind::<dyn kamu::domain::auth::AuthenticationProvider, DummyAuthProvider>();
-                    }
-                }
-            }
         }
         _ => panic!("Unsupported repository scheme: {}", repo_url.scheme()),
     }
-
-    b
-}
-
-/////////////////////////////////////////////////////////////////////////////////////////
-
-fn should_use_multi_tenancy(repo_url: &Url) -> bool {
-    match repo_url.scheme() {
-        "file" => false,
-        "s3" | "s3+http" | "s3+https" => true,
-        _ => panic!("Unsupported repository scheme: {}", repo_url.scheme()),
-    }
 }
 
 /////////////////////////////////////////////////////////////////////////////////////////

diff --git a/src/app/api-server/src/cli_parser.rs b/src/app/api-server/src/cli_parser.rs
@@ -33,6 +33,12 @@ pub fn cli() -> Command {
                 .long("repo-url")
                 .value_parser(value_parse_repo_url)
                 .help("URL of the remote dataset repository"),
+            // TODO: This is temporary and will be removed soon
+            // See: https://github.com/kamu-data/kamu-cli/issues/342
+            Arg::new("multi-tenant")
+                .long("multi-tenant")
+                .action(ArgAction::SetTrue)
+                .help("Indicates that target repo is multi-tenant (for file:// only)"),
         ])
         .subcommands([
             Command::new("run").about("Run the server").args([

diff --git a/src/app/api-server/tests/tests/test_di_graph.rs b/src/app/api-server/tests/tests/test_di_graph.rs
@@ -15,6 +15,7 @@ async fn test_di_graph_validates_local() {
     let mut catalog_builder = kamu_api_server::init_dependencies(
         kamu_api_server::config::ApiServerConfig::default(),
         &url::Url::from_directory_path(tempdir.path()).unwrap(),
+        false,
         tempdir.path(),
     )
     .await;
@@ -61,6 +62,7 @@ async fn test_di_graph_validates_remote() {
     let mut catalog_builder = kamu_api_server::init_dependencies(
         kamu_api_server::config::ApiServerConfig::default(),
         &repo_url,
+        true,
         tmp_repo_dir.path(),
     )
     .await;